LLM inference in C/C++
C++ ggml
CUDA + ggml: add sparse-fa for DSV4/GLM (#27970)
Aman Gupta8e93a9773be6
diff --git a/ggml/include/ggml.h b/ggml/include/ggml.hindex 26f31232f6..b88b7e54a5 100644--- a/ggml/include/ggml.h+++ b/ggml/include/ggml.h@@ -2453,6 +2453,12 @@ extern "C" {GGML_API enum ggml_prec ggml_flash_attn_ext_get_prec(const struct ggml_tensor * a);+ // Use finite mask entries as a sparse K/V set. Set 0 to disable.+ // n_kv_max must bound the number of finite entries in every mask row.+ GGML_API void ggml_flash_attn_ext_set_n_kv_max(+ struct ggml_tensor * a,+ int32_t n_kv_max);+GGML_API void ggml_flash_attn_ext_add_sinks(struct ggml_tensor * a,struct ggml_tensor * sinks);diff --git a/ggml/src/ggml-cuda/fattn-common.cuh b/ggml/src/ggml-cuda/fattn-common.cuhindex e67cc7fdf7..7442bc22af 100644--- a/ggml/src/ggml-cuda/fattn-common.cuh+++ b/ggml/src/ggml-cuda/fattn-common.cuh@@ -718,6 +718,9 @@ static __global__ void flash_attn_mask_to_KV_max(KV_max[sequence*ne31 + jt] = KV_max_sj;}+void ggml_cuda_flash_attn_ext_compact_mask(+ const ggml_tensor * mask, int32_t * indices, int32_t n_kv_max, cudaStream_t stream);+template<int D, int ncols1, int ncols2> // D == head size__launch_bounds__(D, 1)static __global__ void flash_attn_stream_k_fixup_uniform(@@ -972,7 +975,8 @@ static __global__ void flash_attn_combine_results(template <int DV, int ncols1, int ncols2>void launch_fattn(ggml_backend_cuda_context & ctx, ggml_tensor * dst, fattn_kernel_t fattn_kernel, const int nwarps, const size_t nbytes_shared,- const int nbatch_fa, const bool need_f16_K, const bool need_f16_V, const bool stream_k, const int warp_size = WARP_SIZE+ const int nbatch_fa, const bool need_f16_K, const bool need_f16_V, const bool stream_k, const bool use_sparse,+ const int warp_size = WARP_SIZE) {constexpr int ncols = ncols1 * ncols2;@@ -1088,10 +1092,20 @@ void launch_fattn(const int ntiles_z_gqa = ((gqa_ratio + ncols2 - 1) / ncols2);const int ntiles_dst = ntiles_x * ntiles_z_gqa * K->ne[2] * Q->ne[3];+ const int32_t n_kv_max = use_sparse ? ggml_get_op_params_i32(KQV, 4) : 0;+ if (use_sparse) {+ GGML_ASSERT(mask != nullptr);+ GGML_ASSERT(n_kv_max > 0);+ const size_t mask_rows = size_t(mask->ne[1]) * mask->ne[3];++ KV_max.alloc(size_t(n_kv_max) * mask_rows);+ ggml_cuda_flash_attn_ext_compact_mask(mask, KV_max.ptr, n_kv_max, main_stream);+ }+// Optional optimization where the mask is scanned to determine whether part of the calculation can be skipped.// Only worth the overhead if there is at lease one FATTN_KQ_STRIDE x FATTN_KQ_STRIDE square to be skipped or// multiple sequences of possibly different lengths.- if (mask && K->ne[1] % FATTN_KQ_STRIDE == 0 && (Q->ne[1] >= 1024 || Q->ne[3] > 1)) {+ if (!use_sparse && mask && K->ne[1] % FATTN_KQ_STRIDE == 0 && (Q->ne[1] >= 1024 || Q->ne[3] > 1)) {const int64_t s31 = mask->nb[1] / sizeof(half2);const int64_t s33 = mask->nb[3] / sizeof(half2);@@ -1114,7 +1128,8 @@ void launch_fattn(GGML_ASSERT(max_blocks_per_sm > 0);int parallel_blocks = max_blocks_per_sm;- const int ntiles_KV = (K->ne[1] + nbatch_fa - 1) / nbatch_fa; // Max. number of parallel blocks limited by KV cache length.+ const int64_t n_kv = use_sparse ? n_kv_max : K->ne[1];+ const int ntiles_KV = (n_kv + nbatch_fa - 1) / nbatch_fa; // Max. number of parallel blocks limited by KV cache length.dim3 blocks_num;if (stream_k) {@@ -1218,7 +1233,7 @@ void launch_fattn(!stream_k && parallel_blocks > 1 ? dst_tmp.ptr : (float *) KQV->data, dst_tmp_meta.ptr,scale, max_bias, m0, m1, n_head_log2, logit_softcap,Q->ne[0], ne01, Q->ne[2], Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3],- K->ne[0], K->ne[1], K->ne[2], K->ne[3], nb11, nb12, nb13,+ K->ne[0], n_kv, K->ne[2], K->ne[3], nb11, nb12, nb13,nb21, nb22, nb23,mask ? mask->ne[1] : 0, mask ? mask->ne[2] : 0, mask ? mask->ne[3] : 0,mask ? mask->nb[1] : 0, mask ? mask->nb[2] : 0, mask ? mask->nb[3] : 0diff --git a/ggml/src/ggml-cuda/fattn-mma-f16.cuh b/ggml/src/ggml-cuda/fattn-mma-f16.cuhindex 387e70fa14..126a4c4529 100644--- a/ggml/src/ggml-cuda/fattn-mma-f16.cuh+++ b/ggml/src/ggml-cuda/fattn-mma-f16.cuh@@ -350,20 +350,24 @@ static __host__ int ggml_cuda_fattn_mma_get_nstages(const int DKQ, const int DV,return cp_async_available(cc) && ncols2 >= 2 ? ggml_cuda_fattn_mma_get_nstages_target(DKQ, DV, ncols1*ncols2, cc) : 0;}-static constexpr __device__ int ggml_cuda_fattn_mma_get_nstages(const int DKQ, const int DV, const int ncols1, const int ncols2) {+static constexpr __device__ int ggml_cuda_fattn_mma_get_nstages(+ const int DKQ, const int DV, const int ncols1, const int ncols2, const bool use_sparse) {#ifdef CP_ASYNC_AVAILABLE- return ncols2 >= 2 ? ggml_cuda_fattn_mma_get_nstages_target(DKQ, DV, ncols1*ncols2) : 0;+ const int nstages_target = ncols2 >= 2 ? ggml_cuda_fattn_mma_get_nstages_target(DKQ, DV, ncols1*ncols2) : 0;+ // sparse gather is not implemented for multi-stage loading+ return use_sparse && nstages_target > 1 ? 1 : nstages_target;#else- GGML_UNUSED_VARS(DKQ, DV, ncols1, ncols2);+ GGML_UNUSED_VARS(DKQ, DV, ncols1, ncols2, use_sparse);return 0;#endif // CP_ASYNC_AVAILABLE}// -------------------------------------------------------------------------------------------------------------------template<int stride_tile, bool swz, int nwarps, int nbatch_fa, bool use_cp_async, bool oob_check>+template<int stride_tile, bool swz, int nwarps, int nbatch_fa, bool use_cp_async, bool oob_check, bool use_sparse>static __device__ __forceinline__ void flash_attn_ext_f16_load_tile(- const half2 * const __restrict__ KV, half2 * const __restrict__ tile_KV, const int D2, const int stride_KV, const int i_sup) {+ const half2 * const __restrict__ KV, half2 * const __restrict__ tile_KV, const int D2, const int stride_KV,+ const int k_VKQ_0, const int i_sup, const int32_t * const __restrict__ indices) {constexpr int warp_size = ggml_cuda_get_physical_warp_size();// K/V data is loaded with decreasing granularity for D for better memory bandwidth.// The minimum granularity is 16 bytes.@@ -371,7 +375,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_tile(const int chunks_per_row = D2 / h2_per_chunk;if constexpr (use_cp_async) {static_assert(warp_size == 32, "bad warp_size");- static_assert(!oob_check, "OOB check not compatible with cp_async");+ static_assert(!oob_check || use_sparse, "OOB check not compatible with cp_async");constexpr int preload = 64;const unsigned int tile_KV_32 = ggml_cuda_cvta_generic_to_shared(tile_KV);@@ -394,15 +398,24 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_tile(break;}+ int64_t i_KV;+ if constexpr (use_sparse) {+ // padded slots gather row 0, the -inf mask removes their contribution+ const int32_t index = i < i_sup ? indices[k_VKQ_0 + i] : 0;+ i_KV = index >= 0 ? index : 0;+ } else {+ i_KV = k_VKQ_0 + i;+ }+#pragma unrollfor (int k0 = k0_start; k0 < k0_stop; k0 += stride_k) {const int k = k0 + (stride_k == warp_size ? threadIdx.x : threadIdx.x % stride_k);if constexpr (swz) {const int smem_offs_b = ggml_cuda_fattn_smem_swizzle::bytes_rc<stride_tile>(i, k*h2_per_chunk);- cp_async_cg_16<preload>(tile_KV_32 + smem_offs_b, KV + i*stride_KV + k*h2_per_chunk);+ cp_async_cg_16<preload>(tile_KV_32 + smem_offs_b, KV + i_KV*stride_KV + k*h2_per_chunk);} else {- cp_async_cg_16<preload>(tile_KV_32 + i*(stride_tile*sizeof(half2)) + k*16, KV + i*stride_KV + k*h2_per_chunk);+ cp_async_cg_16<preload>(tile_KV_32 + i*(stride_tile*sizeof(half2)) + k*16, KV + i_KV*stride_KV + k*h2_per_chunk);}}}@@ -438,12 +451,17 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_tile(for (int k0 = k0_start; k0 < k0_stop; k0 += stride_k) {const int k = k0 + (stride_k == warp_size ? threadIdx.x : threadIdx.x % stride_k);+ const half2 * src;+ if constexpr (use_sparse) {+ const int32_t index = i < i_sup ? indices[k_VKQ_0 + i] : -1;+ src = index >= 0 ? KV + int64_t(index)*stride_KV + k*h2_per_chunk : zero;+ } else {+ src = !oob_check || i < i_sup ? KV + int64_t(k_VKQ_0 + i)*stride_KV + k*h2_per_chunk : zero;+ }if constexpr (swz) {- ggml_cuda_memcpy_1<16>((char *) tile_KV + ggml_cuda_fattn_smem_swizzle::bytes_rc<stride_tile>(i, k*h2_per_chunk),- !oob_check || i < i_sup ? KV + i*stride_KV + k*h2_per_chunk : zero);+ ggml_cuda_memcpy_1<16>((char *) tile_KV + ggml_cuda_fattn_smem_swizzle::bytes_rc<stride_tile>(i, k*h2_per_chunk), src);} else {- ggml_cuda_memcpy_1<16>(tile_KV + i*stride_tile + k*4,- !oob_check || i < i_sup ? KV + i*stride_KV + k*h2_per_chunk : zero);+ ggml_cuda_memcpy_1<16>(tile_KV + i*stride_tile + k*4, src);}}}@@ -458,14 +476,16 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_tile(}}-template<int ncols1, int nwarps, int nbatch_fa, bool use_cp_async, bool oob_check>+template<int ncols1, int nwarps, int nbatch_fa, bool use_cp_async, bool oob_check, bool use_sparse>static __device__ __forceinline__ void flash_attn_ext_f16_load_mask(const half * const __restrict__ mask_h, half * const __restrict__ tile_mask,- const int stride_mask, const int i_sup, const int j0, const uint3 ne01) {+ const int stride_mask, const int k_VKQ_0, const int i_sup, const int j0, const uint3 ne01,+ const int32_t * const __restrict__ indices) {constexpr int warp_size = ggml_cuda_get_physical_warp_size();if constexpr (use_cp_async) {static_assert(nbatch_fa <= 8*warp_size && nbatch_fa % 8 == 0, "bad nbatch_fa");static_assert(!oob_check, "OOB check incompatible with cp_async");+ static_assert(!use_sparse, "sparse gather incompatible with cp_async");constexpr int preload = nbatch_fa >= 32 ? nbatch_fa * sizeof(half) : 64;constexpr int cols_per_warp = 8*warp_size/nbatch_fa;constexpr int stride_j = nwarps * cols_per_warp;@@ -483,9 +503,9 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_mask(const int i = 8 * (threadIdx.x % (nbatch_fa/8));- cp_async_cg_16<preload>(tile_mask_32 + j_sram*(nbatch_fa*sizeof(half) + 16) + i*sizeof(half), mask_h + int64_t(j_vram)*stride_mask + i);+ cp_async_cg_16<preload>(tile_mask_32 + j_sram*(nbatch_fa*sizeof(half) + 16) + i*sizeof(half), mask_h + int64_t(j_vram)*stride_mask + k_VKQ_0 + i);}- } else if constexpr (oob_check) {+ } else if constexpr (oob_check || use_sparse) {#pragma unrollfor (int j1 = 0; j1 < ncols1; j1 += nwarps) {const int j_sram = j1 + threadIdx.y;@@ -499,7 +519,12 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_mask(for (int i0 = 0; i0 < nbatch_fa; i0 += warp_size) {const int i = i0 + threadIdx.x;- tile_mask[j_sram*(nbatch_fa + 8) + i] = i < i_sup ? mask_h[int64_t(j_vram)*stride_mask + i] : half(0.0f);+ if constexpr (use_sparse) {+ const int32_t index = i < i_sup ? indices[k_VKQ_0 + i] : -1;+ tile_mask[j_sram*(nbatch_fa + 8) + i] = index >= 0 ? mask_h[int64_t(j_vram)*stride_mask + index] : half(-INFINITY);+ } else {+ tile_mask[j_sram*(nbatch_fa + 8) + i] = i < i_sup ? mask_h[int64_t(j_vram)*stride_mask + k_VKQ_0 + i] : half(0.0f);+ }}}} else if constexpr (nbatch_fa < 2*warp_size) {@@ -516,7 +541,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_mask(const int i = threadIdx.x % (warp_size/cols_per_warp);- ggml_cuda_memcpy_1<sizeof(half2)>(tile_mask + j_sram*(nbatch_fa + 8) + 2*i, mask_h + int64_t(j_vram)*stride_mask + 2*i);+ ggml_cuda_memcpy_1<sizeof(half2)>(tile_mask + j_sram*(nbatch_fa + 8) + 2*i, mask_h + int64_t(j_vram)*stride_mask + k_VKQ_0 + 2*i);}} else {#pragma unroll@@ -532,20 +557,21 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_mask(for (int i0 = 0; i0 < nbatch_fa; i0 += 2*warp_size) {const int i = i0 + 2*threadIdx.x;- ggml_cuda_memcpy_1<sizeof(half2)>(tile_mask + j_sram*(nbatch_fa + 8) + i, mask_h + int64_t(j_vram)*stride_mask + i);+ ggml_cuda_memcpy_1<sizeof(half2)>(tile_mask + j_sram*(nbatch_fa + 8) + i, mask_h + int64_t(j_vram)*stride_mask + k_VKQ_0 + i);}}}}template<int DKQ, int DV, int ncols1, int ncols2, int nwarps,- bool use_logit_softcap, bool V_is_K_view, bool needs_fixup, bool is_fixup, bool last_iter, bool oob_check,+ bool use_logit_softcap, bool V_is_K_view, bool use_sparse, bool needs_fixup, bool is_fixup, bool last_iter, bool oob_check,typename T_A_KQ, typename T_B_KQ, typename T_C_KQ, typename T_A_VKQ, typename T_B_VKQ, typename T_C_VKQ>static __device__ __forceinline__ void flash_attn_ext_f16_iter(const float2 * const __restrict__ Q_f2,const half2 * const __restrict__ K_h2,const half2 * const __restrict__ V_h2,const half * const __restrict__ mask_h,+ const int32_t * const __restrict__ indices,float2 * const __restrict__ dstk,float2 * const __restrict__ dstk_fixup,const float scale,@@ -577,7 +603,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(constexpr int nbatch_K2 = ggml_cuda_fattn_mma_get_nbatch_K2(DKQ, DV, ncols);constexpr int nbatch_V2 = ggml_cuda_fattn_mma_get_nbatch_V2(DKQ, DV, ncols);constexpr bool Q_in_reg = ggml_cuda_fattn_mma_get_Q_in_reg (DKQ, DV, ncols);- constexpr int nstages = ggml_cuda_fattn_mma_get_nstages (DKQ, DV, ncols1, ncols2);+ constexpr int nstages = ggml_cuda_fattn_mma_get_nstages (DKQ, DV, ncols1, ncols2, use_sparse);// swizzle the tile stride for K and V based on the batch size.constexpr int stride_tile_K = ggml_cuda_fattn_smem_swizzle::tile_stride(nbatch_K2);@@ -601,13 +627,14 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(constexpr bool use_cp_async = true;cp_async_wait_all();__syncthreads();- flash_attn_ext_f16_load_tile<stride_tile_V, swz_V, nwarps, nbatch_fa, use_cp_async, oob_check>- (V_h2 + int64_t(k_VKQ_0)*stride_V, tile_V, nbatch_V2, stride_V, k_VKQ_sup);+ flash_attn_ext_f16_load_tile<stride_tile_V, swz_V, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse>+ (V_h2, tile_V, nbatch_V2, stride_V, k_VKQ_0, k_VKQ_sup, nullptr);} else {- constexpr bool use_cp_async = nstages == 1;+ // the sparse mask values are gathered per element, always load them synchronously+ constexpr bool use_cp_async = nstages == 1 && !use_sparse;if (ncols2 > 1 || mask_h) {- flash_attn_ext_f16_load_mask<ncols1, nwarps, nbatch_fa, use_cp_async, oob_check>- (mask_h + k_VKQ_0, tile_mask, stride_mask, k_VKQ_sup, jt*ncols1, ne01);+ flash_attn_ext_f16_load_mask<ncols1, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse>+ (mask_h, tile_mask, stride_mask, k_VKQ_0, k_VKQ_sup, jt*ncols1, ne01, indices);}}@@ -620,8 +647,8 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(if constexpr (nstages <= 1) {const int k0_diff = k0_stop - k0_start;constexpr bool use_cp_async = nstages == 1;- flash_attn_ext_f16_load_tile<stride_tile_K, swz_K, nwarps, nbatch_fa, use_cp_async, oob_check>- (K_h2 + int64_t(k_VKQ_0)*stride_K + k0_start, tile_K, k0_diff, stride_K, k_VKQ_sup);+ flash_attn_ext_f16_load_tile<stride_tile_K, swz_K, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse>+ (K_h2 + k0_start, tile_K, k0_diff, stride_K, k_VKQ_0, k_VKQ_sup, indices);if (use_cp_async) {cp_async_wait_all();}@@ -946,6 +973,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(}if constexpr (nstages > 1) {+ static_assert(!use_sparse, "sparse gather not implemented for multi-stage loading");static_assert(!V_is_K_view, "K data reuse not implemented multi-stage loading");// Preload K tile for next iteration:constexpr bool use_cp_async = true;@@ -953,11 +981,11 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(__syncthreads();if (!last_iter) {if (ncols2 > 1 || mask_h) {- flash_attn_ext_f16_load_mask<ncols1, nwarps, nbatch_fa, use_cp_async, oob_check>- (mask_h + k_VKQ_0 + nbatch_fa, tile_mask, stride_mask, k_VKQ_sup, jt*ncols1, ne01);+ flash_attn_ext_f16_load_mask<ncols1, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse>+ (mask_h, tile_mask, stride_mask, k_VKQ_0 + nbatch_fa, k_VKQ_sup, jt*ncols1, ne01, nullptr);}- flash_attn_ext_f16_load_tile<stride_tile_K, swz_K, nwarps, nbatch_fa, use_cp_async, oob_check>- (K_h2 + int64_t(k_VKQ_0 + nbatch_fa)*stride_K, tile_K, nbatch_K2, stride_K, k_VKQ_sup);+ flash_attn_ext_f16_load_tile<stride_tile_K, swz_K, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse>+ (K_h2, tile_K, nbatch_K2, stride_K, k_VKQ_0 + nbatch_fa, k_VKQ_sup, nullptr);}}@@ -972,8 +1000,8 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(const int i0_diff = i0_stop - i0_start;if (!V_is_K_view || i0_stop > 2*nbatch_K2) {constexpr bool use_cp_async = nstages == 1;- flash_attn_ext_f16_load_tile<stride_tile_V, swz_V, nwarps, nbatch_fa, use_cp_async, oob_check>- (V_h2 + int64_t(k_VKQ_0)*stride_V + i0_start/2, tile_V, i0_diff/2, stride_V, k_VKQ_sup);+ flash_attn_ext_f16_load_tile<stride_tile_V, swz_V, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse>+ (V_h2 + i0_start/2, tile_V, i0_diff/2, stride_V, k_VKQ_0, k_VKQ_sup, indices);if (use_cp_async) {cp_async_wait_all();}@@ -1028,7 +1056,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(}}#else- GGML_UNUSED_VARS(Q_f2, K_h2, V_h2, mask_h, dstk, dstk_fixup,+ GGML_UNUSED_VARS(Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup,scale, slope, logit_softcap, ne01, ne02,stride_K, stride_V, stride_mask,tile_Q, tile_K, tile_V, tile_mask,@@ -1126,12 +1154,13 @@ template<int DV, int ncols> struct mma_tile_sizes {};#endif // defined(TURING_MMA_AVAILABLE)-template<int DKQ, int DV, int ncols1, int ncols2, int nwarps, bool use_logit_softcap, bool V_is_K_view, bool needs_fixup, bool is_fixup>+template<int DKQ, int DV, int ncols1, int ncols2, int nwarps, bool use_logit_softcap, bool V_is_K_view, bool use_sparse, bool needs_fixup, bool is_fixup>static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(const float2 * const __restrict__ Q_f2,const half2 * const __restrict__ K_h2,const half2 * const __restrict__ V_h2,const half * const __restrict__ mask_h,+ const int32_t * const __restrict__ indices,const float * const __restrict__ sinks_f,float2 * const __restrict__ dstk,float2 * const __restrict__ dstk_fixup,@@ -1171,7 +1200,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(constexpr int nbatch_V2 = ggml_cuda_fattn_mma_get_nbatch_V2 (DKQ, DV, ncols);constexpr int nbatch_combine = ggml_cuda_fattn_mma_get_nbatch_combine(DKQ, DV, ncols);constexpr bool Q_in_reg = ggml_cuda_fattn_mma_get_Q_in_reg (DKQ, DV, ncols);- constexpr int nstages = ggml_cuda_fattn_mma_get_nstages (DKQ, DV, ncols1, ncols2);+ constexpr int nstages = ggml_cuda_fattn_mma_get_nstages (DKQ, DV, ncols1, ncols2, use_sparse);if (cols_per_warp > ncols) {NO_DEVICE_CODE;@@ -1272,37 +1301,38 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(// Preload mask and K data for first iteration when using cp_async with multiple stages:if constexpr (nstages > 1) {+ static_assert(!use_sparse, "sparse gather not implemented for multi-stage loading");static_assert(nbatch_K2 == DKQ/2, "batching not implemented for multi-stage pipeline");constexpr bool use_cp_async = true;constexpr bool oob_check = false;constexpr int k_VKQ_sup = nbatch_fa;if (ncols2 > 1 || mask_h) {- flash_attn_ext_f16_load_mask<ncols1, nwarps, nbatch_fa, use_cp_async, oob_check>- (mask_h + kb0*nbatch_fa, tile_mask, stride_mask, k_VKQ_sup, jt*ncols1, ne01);+ flash_attn_ext_f16_load_mask<ncols1, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse>+ (mask_h, tile_mask, stride_mask, kb0*nbatch_fa, k_VKQ_sup, jt*ncols1, ne01, nullptr);}- flash_attn_ext_f16_load_tile<stride_tile_K, swz_K, nwarps, nbatch_fa, use_cp_async, oob_check>- (K_h2 + int64_t(kb0)*nbatch_fa*stride_K, tile_K, nbatch_K2, stride_K, k_VKQ_sup);+ flash_attn_ext_f16_load_tile<stride_tile_K, swz_K, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse>+ (K_h2, tile_K, nbatch_K2, stride_K, kb0*nbatch_fa, k_VKQ_sup, nullptr);}// kb0_start is always < kb0_stop so the last iter can be executed unconditionally.- if constexpr (ncols2 == 1) {+ if constexpr (ncols2 == 1 || use_sparse) {constexpr bool oob_check = true;for (; kb0 < kb0_stop-1; ++kb0) {constexpr bool last_iter = false;constexpr int k_VKQ_sup = nbatch_fa;flash_attn_ext_f16_iter- <DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, needs_fixup, is_fixup, last_iter, oob_check,+ <DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup, last_iter, oob_check,T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ>- (Q_f2, K_h2, V_h2, mask_h, dstk, dstk_fixup, scale, slope, logit_softcap,+ (Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup, scale, slope, logit_softcap,ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup);}constexpr bool last_iter = true;const int k_VKQ_sup = ne11 - kb0*nbatch_fa;flash_attn_ext_f16_iter- <DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, needs_fixup, is_fixup, last_iter, oob_check,+ <DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup, last_iter, oob_check,T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ>- (Q_f2, K_h2, V_h2, mask_h, dstk, dstk_fixup, scale, slope, logit_softcap,+ (Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup, scale, slope, logit_softcap,ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup);} else {@@ -1311,18 +1341,18 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(constexpr bool last_iter = false;constexpr int k_VKQ_sup = nbatch_fa;flash_attn_ext_f16_iter- <DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, needs_fixup, is_fixup, last_iter, oob_check,+ <DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup, last_iter, oob_check,T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ>- (Q_f2, K_h2, V_h2, mask_h, dstk, dstk_fixup, scale, slope, logit_softcap,+ (Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup, scale, slope, logit_softcap,ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup);}constexpr bool last_iter = true;constexpr int k_VKQ_sup = nbatch_fa;flash_attn_ext_f16_iter- <DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, needs_fixup, is_fixup, last_iter, oob_check,+ <DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup, last_iter, oob_check,T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ>- (Q_f2, K_h2, V_h2, mask_h, dstk, dstk_fixup, scale, slope, logit_softcap,+ (Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup, scale, slope, logit_softcap,ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup);}@@ -1717,7 +1747,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(}}#else- GGML_UNUSED_VARS(Q_f2, K_h2, V_h2, mask_h, sinks_f, dstk, dstk_fixup,+ GGML_UNUSED_VARS(Q_f2, K_h2, V_h2, mask_h, indices, sinks_f, dstk, dstk_fixup,scale, slope, logit_softcap, ne01, ne02, gqa_ratio,stride_Q1, stride_Q2, stride_K, stride_V, stride_mask,jt, kb0_start, kb0_stop);@@ -1725,7 +1755,13 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(#endif // defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)}-template<int DKQ, int DV, int ncols1, int ncols2, bool use_logit_softcap, bool V_is_K_view>+static constexpr __host__ __device__ bool ggml_cuda_flash_attn_ext_mma_f16_may_use_sparse(+ const int DKQ, const int DV, const int ncols1, const int ncols2) {+ return (DKQ == 512 && DV == 512 && ncols1 == 1 && ncols2 == 8) ||+ (DKQ == 576 && DV == 512 && ncols1 == 1 && ncols2 == 16);+}++template<int DKQ, int DV, int ncols1, int ncols2, bool use_logit_softcap, bool V_is_K_view, bool use_sparse>__launch_bounds__(ggml_cuda_fattn_mma_get_nthreads(DKQ, DV, ncols1*ncols2), ggml_cuda_fattn_mma_get_occupancy(DKQ, DV, ncols1*ncols2))static __global__ void flash_attn_ext_f16(const char * Q_ptr,@@ -1751,14 +1787,15 @@ static __global__ void flash_attn_ext_f16(const int32_t nb31, const int32_t nb32, const int64_t nb33) {ggml_cuda_pdl_sync(); // TODO optimize placement#if defined(FLASH_ATTN_AVAILABLE) && (defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE))- const char * GGML_CUDA_RESTRICT Q = Q_ptr;- const char * GGML_CUDA_RESTRICT K = K_ptr;- const char * GGML_CUDA_RESTRICT V = V_ptr;- const char * GGML_CUDA_RESTRICT mask = mask_ptr;- const char * GGML_CUDA_RESTRICT sinks = sinks_ptr;- const int * GGML_CUDA_RESTRICT KV_max = KV_max_ptr;- float * GGML_CUDA_RESTRICT dst = dst_ptr;- float2 * GGML_CUDA_RESTRICT dst_meta = dst_meta_ptr;+ const char * GGML_CUDA_RESTRICT Q = Q_ptr;+ const char * GGML_CUDA_RESTRICT K = K_ptr;+ const char * GGML_CUDA_RESTRICT V = V_ptr;+ const char * GGML_CUDA_RESTRICT mask = mask_ptr;+ const char * GGML_CUDA_RESTRICT sinks = sinks_ptr;+ const int * GGML_CUDA_RESTRICT KV_max = use_sparse ? nullptr : KV_max_ptr;+ const int * GGML_CUDA_RESTRICT sparse_indices = use_sparse ? KV_max_ptr : nullptr;+ float * GGML_CUDA_RESTRICT dst = dst_ptr;+ float2 * GGML_CUDA_RESTRICT dst_meta = dst_meta_ptr;// Skip unused kernel variants for faster compilation:if (use_logit_softcap && !(DKQ == 128 || DKQ == 256 || DKQ == 512)) {@@ -1769,6 +1806,11 @@ static __global__ void flash_attn_ext_f16(NO_DEVICE_CODE;return;}++ if (!ggml_cuda_flash_attn_ext_mma_f16_may_use_sparse(DKQ, DV, ncols1, ncols2) && use_sparse) {+ NO_DEVICE_CODE;+ return;+ }#ifdef VOLTA_MMA_AVAILABLEif (ncols1*ncols2 < 32) {NO_DEVICE_CODE;@@ -1845,6 +1887,7 @@ static __global__ void flash_attn_ext_f16(const half2 * V_h2 = V_is_K_view ? K_h2 : (const half2 *) (V + nb23*sequence + nb22*z_KV);const float * sinks_f = sinks ? (const float *) sinks + zt_Q : nullptr;+ const int32_t * indices = use_sparse ? sparse_indices + (int64_t(sequence % ne33)*ne31 + jt*ncols1)*ne11 : nullptr;const float slope = ncols2 == 1 ? get_alibi_slope(max_bias, zt_Q, n_head_log2, m0, m1) : 1.0f;@@ -1854,13 +1897,13 @@ static __global__ void flash_attn_ext_f16(constexpr bool is_fixup = false; // All but (potentially) the last iterations write their data to dst rather than the fixup buffer.if (kb0_start == 0) {constexpr bool needs_fixup = false; // CUDA block is working on an entire tile.- flash_attn_ext_f16_process_tile<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, needs_fixup, is_fixup>- (Q_f2, K_h2, V_h2, mask_h, sinks_f, dstk, dst_meta, scale, slope, logit_softcap,+ flash_attn_ext_f16_process_tile<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup>+ (Q_f2, K_h2, V_h2, mask_h, indices, sinks_f, dstk, dst_meta, scale, slope, logit_softcap,ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop);} else {constexpr bool needs_fixup = true; // CUDA block is missing the beginning of a tile.- flash_attn_ext_f16_process_tile<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, needs_fixup, is_fixup>- (Q_f2, K_h2, V_h2, mask_h, sinks_f, dstk, dst_meta, scale, slope, logit_softcap,+ flash_attn_ext_f16_process_tile<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup>+ (Q_f2, K_h2, V_h2, mask_h, indices, sinks_f, dstk, dst_meta, scale, slope, logit_softcap,ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop);}@@ -1891,6 +1934,7 @@ static __global__ void flash_attn_ext_f16(const half2 * V_h2 = V_is_K_view ? K_h2 : (const half2 *) (V + nb23*sequence + nb22*z_KV);const float * sinks_f = sinks ? (const float *) sinks + zt_Q : nullptr;+ const int32_t * indices = use_sparse ? sparse_indices + (int64_t(sequence % ne33)*ne31 + jt*ncols1)*ne11 : nullptr;const float slope = ncols2 == 1 ? get_alibi_slope(max_bias, zt_Q, n_head_log2, m0, m1) : 1.0f;@@ -1900,8 +1944,8 @@ static __global__ void flash_attn_ext_f16(constexpr bool is_fixup = true; // Last index writes its data to fixup buffer to avoid data races with other blocks.constexpr bool needs_fixup = false;- flash_attn_ext_f16_process_tile<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, needs_fixup, is_fixup>- (Q_f2, K_h2, V_h2, mask_h, sinks_f, dstk, dst_meta, scale, slope, logit_softcap,+ flash_attn_ext_f16_process_tile<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup>+ (Q_f2, K_h2, V_h2, mask_h, indices, sinks_f, dstk, dst_meta, scale, slope, logit_softcap,ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop);#elseGGML_UNUSED_VARS(Q_ptr, K_ptr, V_ptr, mask_ptr, sinks_ptr, KV_max_ptr, dst_ptr, dst_meta_ptr, scale,@@ -1917,6 +1961,8 @@ static __global__ void flash_attn_ext_f16(#endif // defined(FLASH_ATTN_AVAILABLE) && (defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE))}+bool ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(ggml_backend_cuda_context & ctx, ggml_tensor * dst);+template <int DKQ, int DV, int ncols1, int ncols2>void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {const ggml_tensor * KQV = dst;@@ -1963,20 +2009,49 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggmlusing fattn_kernel_ptr_t = fattn_kernel_t;#endif // defined(GGML_USE_HIP)fattn_kernel_t fattn_kernel;+ bool use_sparse = false;if (logit_softcap == 0.0f) {constexpr bool use_logit_softcap = false;- fattn_kernel = flash_attn_ext_f16<DKQ, DV, ncols1, ncols2, use_logit_softcap, V_is_K_view>;+#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)+ if constexpr (ggml_cuda_flash_attn_ext_mma_f16_may_use_sparse(DKQ, DV, ncols1, ncols2)) {+ if (ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(ctx, dst)) {+ constexpr bool use_sparse_kernel = true;+ fattn_kernel = flash_attn_ext_f16<DKQ, DV, ncols1, ncols2, use_logit_softcap, V_is_K_view, use_sparse_kernel>;+ use_sparse = true;++ static bool shared_memory_limit_raised[GGML_CUDA_MAX_DEVICES] = {false};+ if (!shared_memory_limit_raised[id]) {+ CUDA_CHECK(cudaFuncSetAttribute(reinterpret_cast<fattn_kernel_ptr_t>(fattn_kernel), cudaFuncAttributeMaxDynamicSharedMemorySize, nbytes_shared_total));+ shared_memory_limit_raised[id] = true;+ }+ } else {+ constexpr bool use_sparse_kernel = false;+ fattn_kernel = flash_attn_ext_f16<DKQ, DV, ncols1, ncols2, use_logit_softcap, V_is_K_view, use_sparse_kernel>;++ static bool shared_memory_limit_raised[GGML_CUDA_MAX_DEVICES] = {false};+ if (!shared_memory_limit_raised[id]) {+ CUDA_CHECK(cudaFuncSetAttribute(reinterpret_cast<fattn_kernel_ptr_t>(fattn_kernel), cudaFuncAttributeMaxDynamicSharedMemorySize, nbytes_shared_total));+ shared_memory_limit_raised[id] = true;+ }+ }+ } else+#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)+ {+ constexpr bool use_sparse_kernel = false;+ fattn_kernel = flash_attn_ext_f16<DKQ, DV, ncols1, ncols2, use_logit_softcap, V_is_K_view, use_sparse_kernel>;#if !defined(GGML_USE_MUSA)- static bool shared_memory_limit_raised[GGML_CUDA_MAX_DEVICES] = {false};- if (!shared_memory_limit_raised[id]) {- CUDA_CHECK(cudaFuncSetAttribute(reinterpret_cast<fattn_kernel_ptr_t>(fattn_kernel), cudaFuncAttributeMaxDynamicSharedMemorySize, nbytes_shared_total));- shared_memory_limit_raised[id] = true;- }+ static bool shared_memory_limit_raised[GGML_CUDA_MAX_DEVICES] = {false};+ if (!shared_memory_limit_raised[id]) {+ CUDA_CHECK(cudaFuncSetAttribute(reinterpret_cast<fattn_kernel_ptr_t>(fattn_kernel), cudaFuncAttributeMaxDynamicSharedMemorySize, nbytes_shared_total));+ shared_memory_limit_raised[id] = true;+ }#endif // !defined(GGML_USE_MUSA)+ }} else {constexpr bool use_logit_softcap = true;- fattn_kernel = flash_attn_ext_f16<DKQ, DV, ncols1, ncols2, use_logit_softcap, V_is_K_view>;+ constexpr bool use_sparse_kernel = false;+ fattn_kernel = flash_attn_ext_f16<DKQ, DV, ncols1, ncols2, use_logit_softcap, V_is_K_view, use_sparse_kernel>;#if !defined(GGML_USE_MUSA)static bool shared_memory_limit_raised[GGML_CUDA_MAX_DEVICES] = {false};@@ -1988,7 +2063,7 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml}launch_fattn<DV, ncols1, ncols2>- (ctx, dst, fattn_kernel, nwarps, nbytes_shared_total, nbatch_fa, true, true, true, warp_size_host);+ (ctx, dst, fattn_kernel, nwarps, nbytes_shared_total, nbatch_fa, true, true, true, use_sparse, warp_size_host);}diff --git a/ggml/src/ggml-cuda/fattn-tile.cuh b/ggml/src/ggml-cuda/fattn-tile.cuhindex d1164b8526..8981ab804c 100644--- a/ggml/src/ggml-cuda/fattn-tile.cuh+++ b/ggml/src/ggml-cuda/fattn-tile.cuh@@ -1163,7 +1163,7 @@ static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggmconst int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc);fattn_kernel_t fattn_kernel = flash_attn_tile<DKQ, DV, cols_per_block/ncols2, ncols2, use_logit_softcap>;launch_fattn<DV, cols_per_block/ncols2, ncols2>- (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, warp_size);+ (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, false, warp_size);return;}}@@ -1179,7 +1179,7 @@ static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggmconst int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc);fattn_kernel_t fattn_kernel = flash_attn_tile<DKQ, DV, cols_per_block/ncols2, ncols2, use_logit_softcap>;launch_fattn<DV, cols_per_block/ncols2, ncols2>- (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, warp_size);+ (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, false, warp_size);return;}}@@ -1191,7 +1191,7 @@ static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggmconst int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc);fattn_kernel_t fattn_kernel = flash_attn_tile<DKQ, DV, cols_per_block/ncols2, ncols2, use_logit_softcap>;launch_fattn<DV, cols_per_block/ncols2, ncols2>- (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, warp_size);+ (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, false, warp_size);return;}}@@ -1203,7 +1203,7 @@ static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggmconst int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc);fattn_kernel_t fattn_kernel = flash_attn_tile<DKQ, DV, cols_per_block/ncols2, ncols2, use_logit_softcap>;launch_fattn<DV, cols_per_block/ncols2, ncols2>- (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, warp_size);+ (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, false, warp_size);return;}}@@ -1215,7 +1215,7 @@ static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggmconst int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc);fattn_kernel_t fattn_kernel = flash_attn_tile<DKQ, DV, cols_per_block/ncols2, ncols2, use_logit_softcap>;launch_fattn<DV, cols_per_block/ncols2, ncols2>- (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, warp_size);+ (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, false, warp_size);return;}}@@ -1226,7 +1226,7 @@ static void launch_fattn_tile_switch_ncols1(ggml_backend_cuda_context & ctx, ggmconst int nbatch_fa = ggml_cuda_fattn_tile_get_nbatch_fa(DKQ, DV, cols_per_block, cc);fattn_kernel_t fattn_kernel = flash_attn_tile<DKQ, DV, cols_per_block/ncols2, ncols2, use_logit_softcap>;launch_fattn<DV, cols_per_block/ncols2, ncols2>- (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, warp_size);+ (ctx, dst, fattn_kernel, nwarps, nbytes_shared, nbatch_fa, true, true, false, false, warp_size);return;}diff --git a/ggml/src/ggml-cuda/fattn-vec.cuh b/ggml/src/ggml-cuda/fattn-vec.cuhindex 69dd936862..519b36b9ff 100644--- a/ggml/src/ggml-cuda/fattn-vec.cuh+++ b/ggml/src/ggml-cuda/fattn-vec.cuh@@ -540,7 +540,7 @@ void ggml_cuda_flash_attn_ext_vec_case_impl(ggml_backend_cuda_context & ctx, ggmconst bool need_f16_K = type_K == GGML_TYPE_F16;const bool need_f16_V = type_V == GGML_TYPE_F16;constexpr size_t nbytes_shared = 0;- launch_fattn<D, cols_per_block, 1>(ctx, dst, fattn_kernel, nwarps, nbytes_shared, D, need_f16_K, need_f16_V, false);+ launch_fattn<D, cols_per_block, 1>(ctx, dst, fattn_kernel, nwarps, nbytes_shared, D, need_f16_K, need_f16_V, false, false);}template <int D, ggml_type type_K, ggml_type type_V>diff --git a/ggml/src/ggml-cuda/fattn.cu b/ggml/src/ggml-cuda/fattn.cuindex ab7a3b297c..ae217fbd9d 100644--- a/ggml/src/ggml-cuda/fattn.cu+++ b/ggml/src/ggml-cuda/fattn.cu@@ -5,11 +5,144 @@#include "fattn-vec.cuh"#include "fattn.cuh"+#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)+__launch_bounds__(256, 1)+static __global__ void flash_attn_mask_to_sparse_indices(+ const half * mask_ptr, int32_t * indices_ptr, const int ne30, const int n_kv_max,+ const int64_t s31, const int64_t s33) {+ ggml_cuda_pdl_sync();++ constexpr int values_per_lane = 8;+ const int tid = threadIdx.x;+ const int warp = tid / WARP_SIZE;+ const int lane = tid % WARP_SIZE;+ const int sequence = blockIdx.y;+ const int query = blockIdx.x;++ const half * mask = mask_ptr + sequence*s33 + query*s31;+ int32_t * indices = indices_ptr + (int64_t(sequence)*gridDim.x + query)*n_kv_max;++ __shared__ int warp_offsets[256/WARP_SIZE];+ __shared__ int row_count;+ __shared__ int chunk_count;++ if (tid == 0) {+ row_count = 0;+ }+ __syncthreads();++ for (int i0 = 0; i0 < ne30; i0 += blockDim.x*values_per_lane) {+ uint32_t selected_warp[values_per_lane];+ int warp_count = 0;+#pragma unroll+ for (int item = 0; item < values_per_lane; ++item) {+ const int i = i0 + (warp*values_per_lane + item)*WARP_SIZE + lane;+ const bool selected = i < ne30 && isfinite(__half2float(mask[i]));+ selected_warp[item] = __ballot_sync(0xFFFFFFFF, selected);+ warp_count += __popc(selected_warp[item]);+ }++ if (lane == 0) {+ warp_offsets[warp] = warp_count;+ }+ __syncthreads();++ if (tid == 0) {+ int offset = 0;+#pragma unroll+ for (int iw = 0; iw < 256/WARP_SIZE; ++iw) {+ const int count = warp_offsets[iw];+ warp_offsets[iw] = offset;+ offset += count;+ }+ chunk_count = offset;+ }+ __syncthreads();++ const uint32_t lane_mask = lane == 0 ? 0 : (1u << lane) - 1;+ int warp_item_offset = 0;+#pragma unroll+ for (int item = 0; item < values_per_lane; ++item) {+ const int i = i0 + (warp*values_per_lane + item)*WARP_SIZE + lane;+ const int dst = row_count + warp_offsets[warp] + warp_item_offset + __popc(selected_warp[item] & lane_mask);+ if ((selected_warp[item] & (uint32_t(1) << lane)) && dst < n_kv_max) {+ indices[dst] = i;+ }+ warp_item_offset += __popc(selected_warp[item]);+ }+ __syncthreads();++ if (tid == 0) {+ row_count += chunk_count;+ }+ __syncthreads();+ }++ const int count = row_count;+ for (int i = count + tid; i < n_kv_max; i += blockDim.x) {+ indices[i] = -1;+ }+ __syncthreads();++ // the dependent grid reads indices, signal once the row is complete+ ggml_cuda_pdl_lc();+}+#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)++void ggml_cuda_flash_attn_ext_compact_mask(+ const ggml_tensor * mask, int32_t * indices, int32_t n_kv_max, cudaStream_t stream) {+#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)+ GGML_UNUSED_VARS(mask, indices, n_kv_max, stream);+ GGML_ABORT("sparse flash attention is only supported on NVIDIA CUDA");+#else+ const int64_t s31 = mask->nb[1] / sizeof(half);+ const int64_t s33 = mask->nb[3] / sizeof(half);+ const dim3 blocks_num(mask->ne[1], mask->ne[3], 1);+ const dim3 block_dim(256, 1, 1);+ const ggml_cuda_kernel_launch_params launch_params(blocks_num, block_dim, 0, stream);+ ggml_cuda_kernel_launch(flash_attn_mask_to_sparse_indices, launch_params,+ (const half *) mask->data, indices, int(mask->ne[0]), n_kv_max, s31, s33);+ CUDA_CHECK(cudaGetLastError());+#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)+}++bool ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {+#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)+ GGML_UNUSED_VARS(ctx, dst);+ return false;+#else+ const ggml_tensor * Q = dst->src[0];+ const ggml_tensor * K = dst->src[1];+ const ggml_tensor * mask = dst->src[3];+ const int cc = ggml_cuda_info().devices[ctx.device].cc;++ float max_bias = 0.0f;+ float logit_softcap = 0.0f;+ memcpy(&max_bias, (const float *) dst->op_params + 1, sizeof(float));+ memcpy(&logit_softcap, (const float *) dst->op_params + 2, sizeof(float));++ const int32_t n_kv_max = ggml_get_op_params_i32(dst, 4);+ return GGML_CUDA_CC_IS_NVIDIA(cc) && turing_mma_available(cc) &&+ mask != nullptr && n_kv_max > 0 && max_bias == 0.0f && logit_softcap == 0.0f &&+ mask->ne[0] == K->ne[1] && mask->ne[1] >= Q->ne[1] && mask->ne[2] == 1 &&+ K->ne[1] >= std::max<int64_t>(4096, 2LL*n_kv_max);+#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)+}+template <int DKQ, int DV, int ncols2>static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;const ggml_tensor * Q = dst->src[0];+#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)+ if constexpr (ggml_cuda_flash_attn_ext_mma_f16_may_use_sparse(DKQ, DV, 1, ncols2)) {+ if (ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(ctx, dst)) {+ ggml_cuda_flash_attn_ext_mma_f16_case<DKQ, DV, 1, ncols2>(ctx, dst);+ return;+ }+ }+#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)+if constexpr (ncols2 <= 8) {if (turing_mma_available(cc) && Q->ne[1] <= 8/ncols2) {ggml_cuda_flash_attn_ext_mma_f16_case<DKQ, DV, 8/ncols2, ncols2>(ctx, dst);diff --git a/ggml/src/ggml.c b/ggml/src/ggml.cindex 3bd3e3fe5e..8dc0945084 100644--- a/ggml/src/ggml.c+++ b/ggml/src/ggml.c@@ -5506,6 +5506,15 @@ enum ggml_prec ggml_flash_attn_ext_get_prec(return (enum ggml_prec) prec_i32;}+void ggml_flash_attn_ext_set_n_kv_max(+ struct ggml_tensor * a,+ int32_t n_kv_max) {+ GGML_ASSERT(a->op == GGML_OP_FLASH_ATTN_EXT);+ GGML_ASSERT(n_kv_max >= 0);++ ggml_set_op_params_i32(a, 4, n_kv_max);+}+void ggml_flash_attn_ext_add_sinks(struct ggml_tensor * a,struct ggml_tensor * sinks) {diff --git a/src/llama-graph.cpp b/src/llama-graph.cppindex 72db486cac..6b7eec14fa 100644--- a/src/llama-graph.cpp+++ b/src/llama-graph.cpp@@ -2540,6 +2540,7 @@ ggml_tensor * llm_graph_context::build_attn_mha(ggml_tensor * kq_mask,ggml_tensor * sinks,ggml_tensor * v_mla,+ int64_t n_kv_max,float kq_scale,int il) const {const bool v_trans = v->nb[1] > v->nb[2];@@ -2577,6 +2578,8 @@ ggml_tensor * llm_graph_context::build_attn_mha(res->add_fused_node({LLM_FUSED_OP_FLASH_ATTN, cur, il});ggml_flash_attn_ext_add_sinks(cur, sinks);+ GGML_ASSERT(n_kv_max >= 0 && n_kv_max <= INT32_MAX);+ ggml_flash_attn_ext_set_n_kv_max(cur, static_cast<int32_t>(n_kv_max));ggml_flash_attn_ext_set_prec (cur, GGML_PREC_F32);if (v_mla) {@@ -2726,7 +2729,7 @@ ggml_tensor * llm_graph_context::build_attn(ggml_tensor * k = k_cur;ggml_tensor * v = v_cur;- ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il);+ ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, 0, kq_scale, il);cb(cur, "kqv_out", il);if (wo) {@@ -2825,7 +2828,7 @@ ggml_tensor * llm_graph_context::build_attn(ggml_tensor * k = mctx_cur->get_k(ctx0, il);ggml_tensor * v = mctx_cur->get_v(ctx0, il);- ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il);+ ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, 0, kq_scale, il);cb(cur, "kqv_out", il);if (inp->self_v_rot) {@@ -2916,7 +2919,7 @@ ggml_tensor * llm_graph_context::build_attn(ggml_tensor * k = mctx_cur->get_k(ctx0, il);ggml_tensor * v = ggml_view_4d(ctx0, k, v_cur->ne[0], k->ne[1], k->ne[2], k->ne[3], k->nb[1], k->nb[2], k->nb[3], 0);- ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il);+ ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, 0, kq_scale, il);cb(cur, "kqv_out", il);if (wo) {@@ -3001,7 +3004,7 @@ ggml_tensor * llm_graph_context::build_attn(ggml_tensor * k = mctx_cur->get_k(ctx0, il);ggml_tensor * v = ggml_view_4d(ctx0, k, v_cur->ne[0], k->ne[1], k->ne[2], k->ne[3], k->nb[1], k->nb[2], k->nb[3], 0);- ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask_top_k, sinks, v_mla, kq_scale, il);+ ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask_top_k, sinks, v_mla, top_k->ne[0], kq_scale, il);cb(cur, "kqv_out", il);if (wo) {@@ -3080,7 +3083,7 @@ ggml_tensor * llm_graph_context::build_attn(ggml_tensor * k = mctx_cur->get_k(ctx0, il);ggml_tensor * v = mctx_cur->get_v(ctx0, il);- ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il);+ ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, 0, kq_scale, il);cb(cur, "kqv_out", il);if (v_rot) {@@ -3151,7 +3154,7 @@ ggml_tensor * llm_graph_context::build_attn(ggml_tensor * k = mctx_cur->get_k(ctx0, il);ggml_tensor * v = ggml_view_4d(ctx0, k, v_cur->ne[0], k->ne[1], k->ne[2], k->ne[3], k->nb[1], k->nb[2], k->nb[3], 0);- ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il);+ ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, 0, kq_scale, il);cb(cur, "kqv_out", il);if (k_rot) {@@ -3210,7 +3213,7 @@ ggml_tensor * llm_graph_context::build_attn(ggml_tensor * k = k_cur;ggml_tensor * v = v_cur;- ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il);+ ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, 0, kq_scale, il);cb(cur, "kqv_out", il);if (wo) {diff --git a/src/llama-graph.h b/src/llama-graph.hindex b388e028cb..dddfdac7b5 100644--- a/src/llama-graph.h+++ b/src/llama-graph.h@@ -1171,6 +1171,7 @@ struct llm_graph_context {ggml_tensor * kq_mask,ggml_tensor * sinks, // [n_head_q]ggml_tensor * v_mla, // [n_embd_head_v_mla, n_embd_head_v, n_head_v]+ int64_t n_kv_max,float kq_scale,int il) const;diff --git a/src/models/deepseek4.cpp b/src/models/deepseek4.cppindex 0157ce7051..222f222492 100644--- a/src/models/deepseek4.cpp+++ b/src/models/deepseek4.cpp@@ -752,7 +752,8 @@ ggml_tensor * llama_model_deepseek4::graph::build_csa_lid_attention(ggml_tensor * kq_mask = ggml_concat(ctx0, raw_mask, csa_mask, 0);cb(kq_mask, "csa_lid_kq_mask", il);- ggml_tensor * out = build_attn_mha(q, k_all, k_all, nullptr, kq_mask, sinks, nullptr, kq_scale, il);+ const int64_t n_kv_max = std::min<int64_t>(raw_mask->ne[0], hparams.n_swa) + top_k->ne[0];+ ggml_tensor * out = build_attn_mha(q, k_all, k_all, nullptr, kq_mask, sinks, nullptr, n_kv_max, kq_scale, il);if (k_rot) {out = llama_mul_mat_hadamard(ctx0, out, k_rot);}@@ -807,7 +808,7 @@ ggml_tensor * llama_model_deepseek4::graph::build_hca_attention(ggml_tensor * kq_mask = ggml_concat(ctx0, raw_mask, hca_mask, 0);cb(kq_mask, "hca_kq_mask", il);- ggml_tensor * out = build_attn_mha(q, k_all, k_all, nullptr, kq_mask, sinks, nullptr, kq_scale, il);+ ggml_tensor * out = build_attn_mha(q, k_all, k_all, nullptr, kq_mask, sinks, nullptr, 0, kq_scale, il);if (k_rot) {out = llama_mul_mat_hadamard(ctx0, out, k_rot);}@@ -843,7 +844,7 @@ ggml_tensor * llama_model_deepseek4::graph::build_raw_attention(ggml_tensor * k = mctx_cur->get_k(ctx0, il);- ggml_tensor * out = build_attn_mha(q, k, k, nullptr, kq_mask, sinks, nullptr, kq_scale, il);+ ggml_tensor * out = build_attn_mha(q, k, k, nullptr, kq_mask, sinks, nullptr, 0, kq_scale, il);if (k_rot) {out = llama_mul_mat_hadamard(ctx0, out, k_rot);}diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cppindex 100a6de426..8f0e47b1fe 100644--- a/src/models/qwen4exp.cpp+++ b/src/models/qwen4exp.cpp@@ -744,7 +744,7 @@ ggml_tensor * llama_model_qwen4exp::graph::build_attn_qsa(ggml_tensor * k = mctx_cur->get_k(ctx0, il);ggml_tensor * v = mctx_cur->get_v(ctx0, il);- ggml_tensor * cur = build_attn_mha(q, k, v, nullptr, kq_mask_top_k, nullptr, nullptr, kq_scale, il);+ ggml_tensor * cur = build_attn_mha(q, k, v, nullptr, kq_mask_top_k, nullptr, nullptr, 0, kq_scale, il);cb(cur, "kqv_out", il);// the rotation is its own inverse, so undo it on the value side of the outputdiff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cppindex 4d9808562d..1b0eaca8f0 100644--- a/tests/test-backend-ops.cpp+++ b/tests/test-backend-ops.cpp@@ -189,6 +189,33 @@ static void init_tensor_kq_mask(ggml_tensor * tensor, float min = -1.0f, float mggml_backend_tensor_set(tensor, data_f16.data(), 0, data_f16.size()*sizeof(ggml_fp16_t));}+static void init_tensor_kq_mask_sparse(ggml_tensor * tensor, int64_t n_kv_max) {+ GGML_ASSERT(tensor->type == GGML_TYPE_F16);+ GGML_ASSERT(n_kv_max > 1 && n_kv_max <= tensor->ne[0]);++ const int64_t ne0 = tensor->ne[0];+ const int64_t nrows = ggml_nrows(tensor);+ std::vector<float> data_f32(ggml_nelements(tensor), -INFINITY);+ std::vector<ggml_fp16_t> data_f16(ggml_nelements(tensor));+ std::vector<int32_t> order(ne0);+ for (int64_t i = 0; i < ne0; ++i) {+ order[i] = i;+ }++ std::mt19937 gen(0x5A17);+ for (int64_t row = 0; row < nrows; ++row) {+ std::shuffle(order.begin(), order.end(), gen);+ const int64_t count = n_kv_max - row % std::min<int64_t>(n_kv_max, 17);+ std::sort(order.begin(), order.begin() + count);+ for (int64_t i = 0; i < count; ++i) {+ data_f32[row*ne0 + order[i]] = -0.03125f * (1 + (i + row) % 7);+ }+ }++ ggml_fp32_to_fp16_row(data_f32.data(), data_f16.data(), data_f16.size());+ ggml_backend_tensor_set(tensor, data_f16.data(), 0, data_f16.size()*sizeof(ggml_fp16_t));+}+// generate a lower triangular matrixstatic void init_tensor_tril(ggml_tensor * tensor, float min = -1.0f, float max = 1.0f) {GGML_ASSERT(tensor->type == GGML_TYPE_F32);@@ -433,6 +460,7 @@ static std::string var_to_str(ggml_scale_mode mode) {#define VARS_TO_STR14(a, b, c, d, e, f, g, h, i, j, k, l, m, n) VAR_TO_STR(a) + "," + VARS_TO_STR13(b, c, d, e, f, g, h, i, j, k, l, m, n)#define VARS_TO_STR15(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o) VAR_TO_STR(a) + "," + VARS_TO_STR14(b, c, d, e, f, g, h, i, j, k, l, m, n, o)#define VARS_TO_STR16(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p) VAR_TO_STR(a) + "," + VARS_TO_STR15(b, c, d, e, f, g, h, i, j, k, l, m, n, o, p)+#define VARS_TO_STR17(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q) VAR_TO_STR(a) + "," + VARS_TO_STR16(b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q)#ifdef GGML_USE_SYCLstatic bool inline _isinf(float f) {@@ -7307,9 +7335,10 @@ struct test_flash_attn_ext : public test_case {std::array<int32_t, 4> permute;const bool kv_view; // create K/V as views of a larger buffer (like a KV cache)const bool v_is_view_of_k;+ const int64_t n_kv_max;std::string vars() override {- return VARS_TO_STR16(hsk, hsv, nh, nr23, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_K, type_V, permute, kv_view, v_is_view_of_k);+ return VARS_TO_STR17(hsk, hsv, nh, nr23, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_K, type_V, permute, kv_view, v_is_view_of_k, n_kv_max);}double max_nmse_err() override {@@ -7326,9 +7355,9 @@ struct test_flash_attn_ext : public test_case {test_flash_attn_ext(int64_t hsk = 128, int64_t hsv = 128, int64_t nh = 32, std::array<int64_t, 2> nr23 = {1, 1}, int64_t kv = 96, int64_t nb = 8,bool mask = true, bool sinks = false, float max_bias = 0.0f, float logit_softcap = 0.0f, ggml_prec prec = GGML_PREC_F32,ggml_type type_K = GGML_TYPE_F16, ggml_type type_V = GGML_TYPE_F16, std::array<int32_t, 4> permute = {0, 1, 2, 3},- bool kv_view = true, bool v_is_view_of_k = false)+ bool kv_view = true, bool v_is_view_of_k = false, int64_t n_kv_max = 0): hsk(hsk), hsv(hsv), nh(nh), nr23(nr23), kv(kv), nb(nb), mask(mask), sinks(sinks), max_bias(max_bias), logit_softcap(logit_softcap), prec(prec),- type_K(type_K), type_V(type_V), permute(permute), kv_view(kv_view), v_is_view_of_k(v_is_view_of_k) {}+ type_K(type_K), type_V(type_V), permute(permute), kv_view(kv_view), v_is_view_of_k(v_is_view_of_k), n_kv_max(n_kv_max) {}ggml_tensor * build_graph(ggml_context * ctx) override {const int64_t hsk_padded = GGML_PAD(hsk, ggml_blck_size(type_K));@@ -7388,6 +7417,7 @@ struct test_flash_attn_ext : public test_case {ggml_tensor * out = ggml_flash_attn_ext(ctx, q, k, v, m, 1.0f/sqrtf(hsk), max_bias, logit_softcap);ggml_flash_attn_ext_add_sinks(out, s);+ ggml_flash_attn_ext_set_n_kv_max(out, n_kv_max);ggml_flash_attn_ext_set_prec (out, prec);ggml_set_name(out, "out");@@ -7400,7 +7430,11 @@ struct test_flash_attn_ext : public test_case {// make the sink values more noticeable in order to trigger a test failure when the implementation is wronginit_tensor_uniform(t, -10.0f, 10.0f);} else if (strcmp(t->name, "m") == 0) {- init_tensor_kq_mask(t);+ if (n_kv_max > 0) {+ init_tensor_kq_mask_sparse(t, n_kv_max);+ } else {+ init_tensor_kq_mask(t);+ }} else {init_tensor_uniform(t);}@@ -10239,6 +10273,14 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1}, 1024, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, true));test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {20, 1}, 1024, 64, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, true));+ // Sparse mask hint: supported decode/prefill layouts and dense fallbacks.+ test_cases.emplace_back(new test_flash_attn_ext(512, 512, 1, { 8, 1}, 4096, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 512));+ test_cases.emplace_back(new test_flash_attn_ext(512, 512, 1, { 8, 2}, 4096, 3, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 768));+ test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {16, 1}, 4096, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, true, 512));+ test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {16, 2}, 4096, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, true, 768));+ test_cases.emplace_back(new test_flash_attn_ext(512, 512, 1, { 8, 1}, 4096, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 2304));+ test_cases.emplace_back(new test_flash_attn_ext(256, 256, 1, { 8, 1}, 4096, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 512));+// more V-is-sub-view-of-K cases: other head shapes, and full views with equal head sizestest_cases.emplace_back(new test_flash_attn_ext(320, 256, 1, {32, 1}, 512, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, true));test_cases.emplace_back(new test_flash_attn_ext(192, 128, 4, {8, 1}, 512, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, true));@@ -10661,6 +10703,12 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0));test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 512, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0));+ // sparse decode at long context+ test_cases.emplace_back(new test_flash_attn_ext(512, 512, 1, { 8, 1}, 49152, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 0));+ test_cases.emplace_back(new test_flash_attn_ext(512, 512, 1, { 8, 1}, 49152, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 2048));+ test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {16, 1}, 49152, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, true, 0));+ test_cases.emplace_back(new test_flash_attn_ext(576, 512, 1, {16, 1}, 49152, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, true, 2048));+// q8_0 KV cases with long context (decode and prompt)test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 128, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0));test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 512, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0));