LLM inference in C/C++
C++ ggml
HIP: remove rocWMMA FlashAttention (#26046)
Johannes Gäßlerfa72aeccb239
diff --git a/CODEOWNERS b/CODEOWNERSindex 9d9530b3e8..929c8380e8 100644--- a/CODEOWNERS+++ b/CODEOWNERS@@ -60,7 +60,6 @@/ggml/src/ggml-cpu/spacemit/ @alex-spacemit/ggml/src/ggml-cuda/ @ggml-org/ggml-cuda/ggml/src/ggml-cuda/vendors/hip.h @IMbackK-/ggml/src/ggml-cuda/fattn-wmma* @IMbackK/ggml/src/ggml-hexagon/ @ggml-org/ggml-hexagon/ggml/src/ggml-hip/ @IMbackK/ggml/src/ggml-et/ @marty1885diff --git a/docs/build.md b/docs/build.mdindex 33ef3ef506..ca086a0be1 100644--- a/docs/build.md+++ b/docs/build.md@@ -361,12 +361,6 @@ You can download it from your Linux distro's package manager or from here: [ROCmNote: `GPU_TARGETS` is optional, omitting it will build the code for all GPUs in the current system.- To enhance flash attention performance on RDNA3+ or CDNA architectures, you can utilize the rocWMMA library by enabling the `-DGGML_HIP_ROCWMMA_FATTN=ON` option. This requires rocWMMA headers to be installed on the build system.-- The rocWMMA library is included by default when installing the ROCm SDK using the `rocm` meta package provided by AMD. Alternatively, if you are not using the meta package, you can install the library using the `rocwmma-dev` or `rocwmma-devel` package, depending on your system's package manager.-- As an alternative, you can manually install the library by cloning it from the official [GitHub repository](https://github.com/ROCm/rocWMMA), checkout the corresponding version tag (e.g. `rocm-6.2.4`) and set `-DCMAKE_CXX_FLAGS="-I<path/to/rocwmma>/library/include/"` in CMake. This also works under Windows despite not officially supported by AMD.-Note that if you get the following error:```clang: error: cannot find ROCm device library; provide its path via '--rocm-path' or '--rocm-device-lib-path', or pass '-nogpulib' to build without ROCm device librarydiff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txtindex e5556c84b2..a766e49ea1 100644--- a/ggml/CMakeLists.txt+++ b/ggml/CMakeLists.txt@@ -216,7 +216,6 @@ option(GGML_HIP "ggml: use HIP"option(GGML_HIP_GRAPHS "ggml: use HIP graph" ON)option(GGML_HIP_RCCL "ggml: use ROCm Collective Comm. Library" OFF)option(GGML_HIP_NO_VMM "ggml: do not try to use HIP VMM" ON)-option(GGML_HIP_ROCWMMA_FATTN "ggml: enable rocWMMA for FlashAttention" OFF)option(GGML_HIP_MMQ_MFMA "ggml: enable MFMA MMA for CDNA in MMQ" ON)option(GGML_HIP_EXPORT_METRICS "ggml: enable kernel perf metrics output" OFF)option(GGML_MUSA_GRAPHS "ggml: use MUSA graph, experimental, unstable" OFF)diff --git a/ggml/src/ggml-cuda/fattn-tile.cu b/ggml/src/ggml-cuda/fattn-tile.cuindex c8281497d1..e563729a2e 100644--- a/ggml/src/ggml-cuda/fattn-tile.cu+++ b/ggml/src/ggml-cuda/fattn-tile.cu@@ -1,6 +1,5 @@#include "common.cuh"#include "fattn-tile.cuh"-#include "fattn-wmma-f16.cuh"void ggml_cuda_flash_attn_ext_tile(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {const ggml_tensor * K = dst->src[1];diff --git a/ggml/src/ggml-cuda/fattn-tile.cuh b/ggml/src/ggml-cuda/fattn-tile.cuhindex 3e07a9f7e0..d1164b8526 100644--- a/ggml/src/ggml-cuda/fattn-tile.cuh+++ b/ggml/src/ggml-cuda/fattn-tile.cuh@@ -1,6 +1,5 @@#include "common.cuh"#include "fattn-common.cuh"-#include "fattn-wmma-f16.cuh"// nbatch_fa == number of KQ rows to process per iteration// nbatch_K == number of K columns to load in parallel for KQ calculation@@ -825,12 +824,7 @@ static __global__ void flash_attn_tile(// Skip unused kernel variants for faster compilation:- if (-#ifdef GGML_USE_WMMA_FATTN- (ncols2 != 1 && DV != 40 && DV != 72 && DV != 512) ||-#endif // GGML_USE_WMMA_FATTN- (use_logit_softcap && !(DV == 128 || DV == 256 || DV == 512))- ) {+ if ((use_logit_softcap && !(DV == 128 || DV == 256 || DV == 512))) {GGML_UNUSED_VARS(Q, K, V, mask, sinks, KV_max, dst, dst_meta, scale,max_bias, m0, m1, n_head_log2, logit_softcap,ne00, ne01, ne02, ne03,diff --git a/ggml/src/ggml-cuda/fattn-wmma-f16.cu b/ggml/src/ggml-cuda/fattn-wmma-f16.cudeleted file mode 100644index 6850716fc0..0000000000--- a/ggml/src/ggml-cuda/fattn-wmma-f16.cu+++ /dev/null@@ -1,705 +0,0 @@-// Old and deprecated WMMA FlashAttention implementation.-// It is still needed for Volta since the memory layout of NVIDIA tensor cores changed with Turing.-// Long-term the WMMA code should be replaced with a dedicated Volta implementation.--#include "common.cuh"-#include "fattn-common.cuh"-#include "fattn-wmma-f16.cuh"--#ifdef GGML_USE_WMMA_FATTN-#if !defined(GGML_USE_HIP)-#include <mma.h>-#if defined(GGML_USE_MUSA)-namespace wmma = mtmusa::wmma;-#else // GGML_USE_MUSA-namespace wmma = nvcuda::wmma;-#endif // GGML_USE_MUSA-#elif defined(GGML_USE_HIP)-#include <rocwmma/rocwmma.hpp>-namespace wmma = rocwmma;-#endif // !defined(GGML_USE_HIP)-#endif // GGML_USE_WMMA_FATTN--// D == head size, VKQ_stride == num VKQ rows calculated in parallel:-template<int D, int ncols, int nwarps, int VKQ_stride, typename KQ_acc_t, bool use_logit_softcap>-__launch_bounds__(nwarps*ggml_cuda_get_physical_warp_size(), 1)-static __global__ void flash_attn_ext_f16(- const char * Q_ptr,- const char * K_ptr,- const char * V_ptr,- const char * mask_ptr,- const char * sinks_ptr,- const int * KV_max_ptr,- float * dst_ptr,- float2 * dst_meta_ptr,- const float scale,- const float max_bias,- const float m0,- const float m1,- const uint32_t n_head_log2,- const float logit_softcap,- const int32_t ne00, const uint3 ne01, const int32_t ne02, const int32_t ne03,- const int32_t nb01, const int32_t nb02, const int32_t nb03,- const int32_t ne10, const int32_t ne11, const int32_t ne12, const int32_t ne13,- const int32_t nb11, const int32_t nb12, const int64_t nb13,- const int32_t nb21, const int32_t nb22, const int64_t nb23,- const int32_t ne31, const int32_t ne32, const int32_t ne33,- const int32_t nb31, const int32_t nb32, const int64_t nb33) {-#if defined(FLASH_ATTN_AVAILABLE) && (defined(GGML_HIP_ROCWMMA_FATTN) && defined(GGML_USE_WMMA_FATTN))- const char * GGML_CUDA_RESTRICT Q = Q_ptr;- const char * GGML_CUDA_RESTRICT K = K_ptr;- const char * GGML_CUDA_RESTRICT V = V_ptr;- const char * GGML_CUDA_RESTRICT mask = mask_ptr;- const char * GGML_CUDA_RESTRICT sinks = sinks_ptr;- const int * GGML_CUDA_RESTRICT KV_max = KV_max_ptr;- float * GGML_CUDA_RESTRICT dst = dst_ptr;- float2 * GGML_CUDA_RESTRICT dst_meta = dst_meta_ptr;- // Skip unused kernel variants for faster compilation:- if (use_logit_softcap && !(D == 128 || D == 256)) {- NO_DEVICE_CODE;- return;- }-- //In this kernel Q, K, V are matrices while i, j, k are matrix indices.-- constexpr int warp_size = ggml_cuda_get_physical_warp_size();-- const int ic0 = ncols*blockIdx.x; // Index of the first Q/QKV column to work on.-- static_assert(D <= FATTN_KQ_STRIDE, "D must be <= FATTN_KQ_STRIDE.");- static_assert(ncols == 8 || ncols % 16 == 0, "ncols must be 8 or a multiple of 16.");- constexpr int frag_m = ncols == 8 ? 32 : 16;- constexpr int frag_n = ncols == 8 ? 8 : 16;- static_assert(D % frag_m == 0, "If ncols == 8 then D % frag_m must be 0.");-#if defined(GGML_USE_HIP) && HIP_VERSION >= 60500000- typedef wmma::fragment<wmma::matrix_a, frag_m, frag_n, 16, _Float16, wmma::row_major> frag_a_K;- typedef wmma::fragment<wmma::matrix_a, frag_m, frag_n, 16, _Float16, wmma::col_major> frag_a_V;- typedef wmma::fragment<wmma::matrix_b, frag_m, frag_n, 16, _Float16, wmma::col_major> frag_b;- typedef wmma::fragment<wmma::accumulator, frag_m, frag_n, 16, KQ_acc_t> frag_c_KQ;- typedef wmma::fragment<wmma::accumulator, frag_m, frag_n, 16, _Float16> frag_c_VKQ;-#else- typedef wmma::fragment<wmma::matrix_a, frag_m, frag_n, 16, half, wmma::row_major> frag_a_K;- typedef wmma::fragment<wmma::matrix_a, frag_m, frag_n, 16, half, wmma::col_major> frag_a_V;- typedef wmma::fragment<wmma::matrix_b, frag_m, frag_n, 16, half, wmma::col_major> frag_b;- typedef wmma::fragment<wmma::accumulator, frag_m, frag_n, 16, KQ_acc_t> frag_c_KQ;- typedef wmma::fragment<wmma::accumulator, frag_m, frag_n, 16, half> frag_c_VKQ;-#endif-- constexpr int KQ_stride_tc = nwarps*frag_m; // Number of KQ rows calculated in parallel.- constexpr int VKQ_ratio = KQ_stride_tc/VKQ_stride; // Number of parallel VKQ accumulators needed to keep all warps busy.- static_assert(VKQ_ratio <= nwarps, "VKQ_ratio must be <= nwarps.");-- // Pad internal representation of KQ, KQV to reduce shared memory bank conflicts:- constexpr int D_padded = D + 8;- constexpr int kqs_padded = FATTN_KQ_STRIDE + 8;- constexpr int kqar = sizeof(KQ_acc_t)/sizeof(half);-- ggml_cuda_pdl_sync();- const int sequence = blockIdx.z / ne02;- const int head = blockIdx.z - sequence*ne02;- const int gqa_ratio = ne02 / ne12; // With grouped query attention there are > 1 Q matrices per K, V matrix.- const float * Q_f = (const float *) (Q + nb03* sequence + nb02* head + nb01*ic0);- const half * K_h = (const half *) (K + nb13* sequence + nb12*(head / gqa_ratio));- const half * V_h = (const half *) (V + nb13* sequence + nb12*(head / gqa_ratio)); // K and V have same shape- const half * maskh = (const half *) (mask + nb33*(sequence % ne33) + nb31*ic0);- const half2 * mask2 = (const half2 *) maskh;- const float * sinksf = (const float *) sinks;-- const int stride_Q = nb01 / sizeof(float);- const int stride_KV = nb11 / sizeof(half);-- const float slopef = get_alibi_slope(max_bias, head, n_head_log2, m0, m1);- const half slopeh = __float2half(slopef);- const half2 slope2 = make_half2(slopef, slopef);-- const half2 logit_softcap_2 = make_half2(logit_softcap, logit_softcap);-- frag_b Q_b[D/16][ncols/frag_n];-- // A single buffer for temporarily holding tiles of KQ and VKQ parts:- constexpr int mem_KQ = ncols*kqs_padded*kqar;- constexpr int mem_VKQ_parts = VKQ_ratio*ncols*D_padded;- __shared__ half KQ[mem_KQ >= mem_VKQ_parts ? mem_KQ : mem_VKQ_parts];- float * KQ_f = (float *) KQ;- half2 * KQ2 = (half2 *) KQ;-- float KQ_rowsum_f[ncols/nwarps] = {0.0f};- float KQ_max_f[ncols/nwarps];- float KQ_max_scale_f[ncols/nwarps] = {0.0f};--#pragma unroll- for (int j = 0; j < ncols/nwarps; ++j) {- KQ_max_f[j] = -FLT_MAX/2.0f;- }-- half2 KQ_rowsum_h2[ncols/nwarps] = {{0.0f, 0.0f}};- half2 KQ_max_h2[ncols/nwarps];- half2 KQ_max_scale_h2[ncols/nwarps] = {{0.0f, 0.0f}};--#pragma unroll- for (int j = 0; j < ncols/nwarps; ++j) {- KQ_max_h2[j] = make_half2(-HALF_MAX_HALF, -HALF_MAX_HALF);- }-- __shared__ half VKQ[ncols*D_padded]; // Accumulator for final VKQ slice.- half2 * VKQ2 = (half2 *) VKQ;--#if defined(GGML_USE_HIP) && HIP_VERSION >= 60500000- const _Float16 * K_h_f16 = reinterpret_cast<const _Float16 *>(K_h);- const _Float16 * V_h_f16 = reinterpret_cast<const _Float16 *>(V_h);- _Float16 * KQ_f16 = reinterpret_cast<_Float16 *>(KQ);- _Float16 * VKQ_f16 = reinterpret_cast<_Float16 *>(VKQ);-#else- const half * K_h_f16 = K_h;- const half * V_h_f16 = V_h;- half * KQ_f16 = KQ;- half * VKQ_f16 = VKQ;-#endif--#pragma unroll- for (int j0 = 0; j0 < ncols; j0 += nwarps) {- const int j = j0 + threadIdx.y;-#pragma unroll- for (int i0 = 0; i0 < D/2; i0 += warp_size) {- const int i = i0 + threadIdx.x;- if (i0 + warp_size > D/2 && i >= D/2) {- break;- }- VKQ2[j*(D_padded/2) + i] = make_half2(0.0f, 0.0f);- }- }-- // Convert Q to half and apply scale, temporarily store in KQ:-#pragma unroll- for (int j0 = 0; j0 < ncols; j0 += nwarps) {- const int j = j0 + threadIdx.y;-#pragma unroll- for (int i0 = 0; i0 < D; i0 += warp_size) {- const int i = i0 + threadIdx.x;- if (i0 + warp_size > D && i >= D) {- break;- }- KQ[j*D_padded + i] = ic0 + j < int(ne01.z) ? Q_f[j*stride_Q + i] * scale : 0.0f;- }- }-- __syncthreads();-- // Load Q into tensor core fragments/registers since it will be used frequently:-#pragma unroll- for (int i0 = 0; i0 < D; i0 += 16) {-#pragma unroll- for (int j0 = 0; j0 < ncols; j0 += frag_n) {- wmma::load_matrix_sync(Q_b[i0/16][j0/frag_n], KQ_f16 + j0*D_padded + i0, D_padded);- }- }-- __syncthreads();-- // Iterate over ne11 == previous tokens:- const int k_VKQ_max = KV_max ? KV_max[sequence*gridDim.x + blockIdx.x] : ne11;- for (int k_VKQ_0 = blockIdx.y*FATTN_KQ_STRIDE; k_VKQ_0 < k_VKQ_max; k_VKQ_0 += gridDim.y*FATTN_KQ_STRIDE) {- // Calculate tile of KQ:-#pragma unroll- for (int i_KQ_0 = 0; i_KQ_0 < FATTN_KQ_STRIDE; i_KQ_0 += KQ_stride_tc) {- frag_c_KQ KQ_c[ncols/frag_n];-#pragma unroll- for (int j = 0; j < ncols/frag_n; ++j) {- wmma::fill_fragment(KQ_c[j], static_cast<KQ_acc_t>(0.0f));- }-#pragma unroll- for (int k_KQ_0 = 0; k_KQ_0 < D; k_KQ_0 += 16) {- frag_a_K K_a;- wmma::load_matrix_sync(K_a, K_h_f16 + int64_t(k_VKQ_0 + i_KQ_0 + frag_m*threadIdx.y)*stride_KV + k_KQ_0, stride_KV);-#pragma unroll- for (int j = 0; j < ncols/frag_n; ++j) {- wmma::mma_sync(KQ_c[j], K_a, Q_b[k_KQ_0/16][j], KQ_c[j]);- }- }-#pragma unroll- for (int j0 = 0; j0 < ncols; j0 += frag_n) {- wmma::store_matrix_sync((KQ_acc_t *) KQ + j0*kqs_padded + i_KQ_0 + frag_m*threadIdx.y, KQ_c[j0/frag_n], kqs_padded, wmma::mem_col_major);- }- }-- __syncthreads();-- // Calculate softmax for each KQ column using the current max. value.- // The divisor is stored in KQ_rowsum and will be applied at the end.-#pragma unroll- for (int j0 = 0; j0 < ncols; j0 += nwarps) {- const int j = j0 + threadIdx.y;-- if (std::is_same<KQ_acc_t, float>::value) {- float KQ_f_tmp[FATTN_KQ_STRIDE / warp_size];-#pragma unroll- for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += warp_size) {- const int k = k0 + threadIdx.x;-- KQ_f_tmp[k0/warp_size] = KQ_f[j*kqs_padded + k];-- if (use_logit_softcap) {- KQ_f_tmp[k0/warp_size] = logit_softcap*tanhf(KQ_f_tmp[k0/warp_size]);- }- }-- float KQ_max_new = KQ_max_f[j0/nwarps];-#pragma unroll- for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += warp_size) {- const int k = k0 + threadIdx.x;-- KQ_f_tmp[k0/warp_size] += mask && ic0 + j < int(ne01.z) ?- __half2float(slopeh*maskh[j*(nb31/sizeof(half)) + k_VKQ_0 + k]) : 0.0f;- KQ_max_new = max(KQ_max_new, KQ_f_tmp[k0/warp_size] + FATTN_KQ_MAX_OFFSET);- }- KQ_max_new = warp_reduce_max<warp_size>(KQ_max_new);-- const float diff = KQ_max_f[j0/nwarps] - KQ_max_new;- KQ_max_scale_f[j0/nwarps] = expf(diff);- if (diff <= SOFTMAX_FTZ_THRESHOLD) {- KQ_max_scale_f[j0/nwarps] = 0.0f;- }- KQ_max_f[j0/nwarps] = KQ_max_new;-- float KQ_rowsum_add = 0.0f;-#pragma unroll- for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += warp_size) {- const int k = k0 + threadIdx.x;-- const float diff = KQ_f_tmp[k0/warp_size] - KQ_max_f[j0/nwarps];- KQ_f_tmp[k0/warp_size] = expf(diff);- if (diff <= SOFTMAX_FTZ_THRESHOLD) {- KQ_f_tmp[k0/warp_size] = 0.0f;- }- KQ_rowsum_add += KQ_f_tmp[k0/warp_size];- KQ[j*(kqar*kqs_padded) + k] = KQ_f_tmp[k0/warp_size];- }- KQ_rowsum_add = warp_reduce_sum<warp_size>(KQ_rowsum_add);-- // Scale previous KQ_rowsum to account for a potential increase in KQ_max:- KQ_rowsum_f[j0/nwarps] = KQ_max_scale_f[j0/nwarps]*KQ_rowsum_f[j0/nwarps] + KQ_rowsum_add;- } else {- half2 KQ2_tmp[FATTN_KQ_STRIDE/(2*warp_size)];-#pragma unroll- for (int k0 = 0; k0 < FATTN_KQ_STRIDE/2; k0 += warp_size) {- const int k = k0 + threadIdx.x;-- KQ2_tmp[k0/warp_size] = KQ2[j*(kqs_padded/2) + k];-- if (use_logit_softcap) {- // There is no dedicated tangens hyperbolicus function for half2.- KQ2_tmp[k0/warp_size] = h2exp(KQ2_tmp[k0/warp_size]*make_half2(2.0f, 2.0f));- KQ2_tmp[k0/warp_size] = (KQ2_tmp[k0/warp_size] - make_half2(1.0f, 1.0f))- /(KQ2_tmp[k0/warp_size] + make_half2(1.0f, 1.0f));-- KQ2_tmp[k0/warp_size] *= logit_softcap_2;- }- }-- half2 KQ_max_new = KQ_max_h2[j0/nwarps];-#pragma unroll- for (int k0 = 0; k0 < FATTN_KQ_STRIDE/2; k0 += warp_size) {- const int k = k0 + threadIdx.x;-- KQ2_tmp[k0/warp_size] += mask && ic0 + j < int(ne01.z) ? slope2*mask2[(j*ne11 + k_VKQ_0)/2 + k] : make_half2(0.0f, 0.0f);- KQ_max_new = ggml_cuda_hmax2(KQ_max_new, KQ2_tmp[k0/warp_size]);- }- KQ_max_new = __half2half2(warp_reduce_max<warp_size>(ggml_cuda_hmax(__low2half(KQ_max_new), __high2half(KQ_max_new))));- const half2 diff = KQ_max_h2[j0/nwarps] - KQ_max_new;- KQ_max_scale_h2[j0/nwarps] = h2exp(diff);- const uint32_t ftz_mask = __hgt2_mask(diff, make_half2(SOFTMAX_FTZ_THRESHOLD, SOFTMAX_FTZ_THRESHOLD));- *((uint32_t *) &KQ_max_scale_h2[j0/nwarps]) &= ftz_mask;- KQ_max_h2[j0/nwarps] = KQ_max_new;-- half2 KQ_rowsum_add = make_half2(0.0f, 0.0f);-#pragma unroll- for (int k0 = 0; k0 < FATTN_KQ_STRIDE/2; k0 += warp_size) {- const int k = k0 + threadIdx.x;-- const half2 diff = KQ2_tmp[k0/warp_size] - KQ_max_h2[j0/nwarps];- KQ2_tmp[k0/warp_size] = h2exp(diff);- const uint32_t ftz_mask = __hgt2_mask(diff, make_half2(SOFTMAX_FTZ_THRESHOLD, SOFTMAX_FTZ_THRESHOLD));- *((uint32_t *) &KQ2_tmp[k0/warp_size]) &= ftz_mask;- KQ_rowsum_add += KQ2_tmp[k0/warp_size];- KQ2[j*(kqs_padded/2) + k] = KQ2_tmp[k0/warp_size];- }- KQ_rowsum_add = warp_reduce_sum<warp_size>(KQ_rowsum_add);-- // Scale previous KQ_rowsum to account for a potential increase in KQ_max:- KQ_rowsum_h2[j0/nwarps] = KQ_max_scale_h2[j0/nwarps]*KQ_rowsum_h2[j0/nwarps] + KQ_rowsum_add;- }- }-- __syncthreads();-- frag_b KQ_b[FATTN_KQ_STRIDE/(VKQ_ratio*16)][ncols/frag_n];-#pragma unroll- for (int j0 = 0; j0 < ncols; j0 += frag_n) {-#pragma unroll- for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += VKQ_ratio*16) {- const int k = k0 + (threadIdx.y % VKQ_ratio)*16;- wmma::load_matrix_sync(- KQ_b[k0/(VKQ_ratio*16)][j0/frag_n],- KQ_f16 + j0*(kqar*kqs_padded) + k,- kqar*kqs_padded);- }- }-- frag_c_VKQ VKQ_c[D/VKQ_stride][ncols/frag_n];-#pragma unroll- for (int i_VKQ_0 = 0; i_VKQ_0 < D; i_VKQ_0 += VKQ_stride) {-#pragma unroll- for (int j = 0; j < ncols/frag_n; ++j) {- wmma::fill_fragment(VKQ_c[i_VKQ_0/VKQ_stride][j], static_cast<half>(0.0f));- }--#pragma unroll- for (int k0 = 0; k0 < FATTN_KQ_STRIDE; k0 += VKQ_ratio*16) {- const int k = k0 + (threadIdx.y % VKQ_ratio)*16;-- frag_a_V v_a;- wmma::load_matrix_sync(v_a, V_h_f16 + int64_t(k_VKQ_0 + k)*stride_KV + i_VKQ_0 + frag_m*(threadIdx.y/VKQ_ratio), stride_KV);-#pragma unroll- for (int j = 0; j < ncols/frag_n; ++j) {- wmma::mma_sync(VKQ_c[i_VKQ_0/VKQ_stride][j], v_a, KQ_b[k0/(VKQ_ratio*16)][j], VKQ_c[i_VKQ_0/VKQ_stride][j]);- }- }- }-- __syncthreads();-- const int offset_k = (threadIdx.y % VKQ_ratio) * (ncols*D_padded);-#pragma unroll- for (int i_KQ_0 = 0; i_KQ_0 < D; i_KQ_0 += VKQ_stride) {-#pragma unroll- for (int j0 = 0; j0 < ncols; j0 += frag_n) {- wmma::store_matrix_sync(- KQ_f16 + offset_k + j0*D_padded + i_KQ_0 + frag_m*(threadIdx.y/VKQ_ratio),- VKQ_c[i_KQ_0/VKQ_stride][j0/frag_n],- D_padded, wmma::mem_col_major);- }- }-- __syncthreads();--#pragma unroll- for (int j0 = 0; j0 < ncols; j0 += nwarps) {- const int j = j0 + threadIdx.y;-- half2 VKQ_scale;- if (std::is_same<KQ_acc_t, float>::value) {- VKQ_scale = make_half2(KQ_max_scale_f[j0/nwarps], KQ_max_scale_f[j0/nwarps]);- } else {- VKQ_scale = KQ_max_scale_h2[j0/nwarps];- }--#pragma unroll- for (int i0 = 0; i0 < D/2; i0 += warp_size) {- const int i = i0 + threadIdx.x;- if (i0 + warp_size > D/2 && i >= D/2) {- break;- }-- half2 VKQ_add = make_half2(0.0f, 0.0f);-#pragma unroll- for (int l = 0; l < VKQ_ratio; ++l) {- VKQ_add += KQ2[l*(ncols*D_padded/2) + j*(D_padded/2) + i];- }- VKQ2[j*(D_padded/2) + i] = VKQ_scale*VKQ2[j*(D_padded/2) + i] + VKQ_add;- }- }-- __syncthreads();- }-- // Apply attention sinks- if (sinksf && blockIdx.y == 0) {- const float sinkf = sinksf[head];- const half sinkh = __float2half(sinkf);--#pragma unroll- for (int j0 = 0; j0 < ncols; j0 += nwarps) {- const int j = j0 + threadIdx.y;-- if (std::is_same<KQ_acc_t, float>::value) {- float kqmax_new = fmaxf(KQ_max_f[j0/nwarps], sinkf);-- const float KQ_max_scale = expf(KQ_max_f[j0/nwarps] - kqmax_new);- KQ_max_f[j0/nwarps] = kqmax_new;-- KQ_rowsum_f[j0/nwarps] = KQ_rowsum_f[j0/nwarps] * KQ_max_scale + expf(sinkf - KQ_max_f[j0/nwarps]);-- const half2 scale_h2 = make_half2(KQ_max_scale, KQ_max_scale);-#pragma unroll- for (int i0 = 0; i0 < D/2; i0 += warp_size) {- const int i = i0 + threadIdx.x;- if (i0 + warp_size > D/2 && i >= D/2) break;- VKQ2[j*(D_padded/2) + i] *= scale_h2;- }- } else {- half kqmax_old = __low2half(KQ_max_h2[j0/nwarps]);- half kqmax_new = fmaxf(kqmax_old, sinkh);- KQ_max_h2[j0/nwarps] = __half2half2(kqmax_new);-- const half KQ_max_scale_h = hexp(kqmax_old - kqmax_new);- const half2 KQ_max_scale = __half2half2(KQ_max_scale_h);-- KQ_rowsum_h2[j0/nwarps] = KQ_rowsum_h2[j0/nwarps] * KQ_max_scale;- const half val = hexp(sinkh - kqmax_new);- KQ_rowsum_h2[j0/nwarps].x = __hadd(KQ_rowsum_h2[j0/nwarps].x, val);--#pragma unroll- for (int i0 = 0; i0 < D/2; i0 += warp_size) {- const int i = i0 + threadIdx.x;- if (i0 + warp_size > D/2 && i >= D/2) break;- VKQ2[j*(D_padded/2) + i] *= KQ_max_scale;- }- }- }-- __syncthreads();- }-#pragma unroll- for (int j0 = 0; j0 < ncols; j0 += nwarps) {- const int j_VKQ = j0 + threadIdx.y;- if (ic0 + j_VKQ >= int(ne01.z)) {- return;- }-- float KQ_rowsum_j;- if (std::is_same<KQ_acc_t, float>::value) {- KQ_rowsum_j = KQ_rowsum_f[j0/nwarps];- } else {- KQ_rowsum_j = __low2float(KQ_rowsum_h2[j0/nwarps]) + __high2float(KQ_rowsum_h2[j0/nwarps]);- }-- const int j_dst_unrolled = ((sequence*int(ne01.z) + ic0 + j_VKQ)*ne02 + head)*gridDim.y + blockIdx.y;--#pragma unroll- for (int i0 = 0; i0 < D; i0 += warp_size) {- const int i = i0 + threadIdx.x;- if (i0 + warp_size > D && i >= D) {- break;- }- float dst_val = VKQ[j_VKQ*D_padded + i];- if (gridDim.y == 1) {- dst_val /= KQ_rowsum_j;- }- dst[j_dst_unrolled*D + i] = dst_val;- }-- if (gridDim.y == 1 || threadIdx.x != 0) {- continue;- }-- float2 dst_meta_val;- if (std::is_same<KQ_acc_t, float>::value) {- dst_meta_val.x = KQ_max_f[j0/nwarps];- } else {- dst_meta_val.x = __low2float(KQ_max_h2[j0/nwarps]);- }- dst_meta_val.y = KQ_rowsum_j;- dst_meta[j_dst_unrolled] = dst_meta_val;- }-#else- GGML_UNUSED_VARS(Q_ptr, K_ptr, V_ptr, mask_ptr, sinks_ptr, KV_max_ptr, dst_ptr, dst_meta_ptr, scale,- max_bias, m0, m1, n_head_log2, logit_softcap,- ne00, ne01, ne02, ne03,- nb01, nb02, nb03,- ne10, ne11, ne12, ne13,- nb11, nb12, nb13,- nb21, nb22, nb23,- ne31, ne32, ne33,- nb31, nb32, nb33);- NO_DEVICE_CODE;-#endif // defined(FLASH_ATTN_AVAILABLE) && (defined(GGML_HIP_ROCWMMA_FATTN) && defined(GGML_USE_WMMA_FATTN))-}--constexpr int get_max_power_of_2(int x) {- return x % 2 == 0 ? 2*get_max_power_of_2(x/2) : 1;-}--static_assert(get_max_power_of_2(1) == 1, "Test failed.");-static_assert(get_max_power_of_2(2) == 2, "Test failed.");-static_assert(get_max_power_of_2(4) == 4, "Test failed.");-static_assert(get_max_power_of_2(6) == 2, "Test failed.");--// Number of VKQ rows calculated in parallel:-constexpr int get_VKQ_stride(int D, int nwarps, int frag_m) {- return (get_max_power_of_2(D/frag_m) < nwarps ? get_max_power_of_2(D/frag_m) : nwarps)*frag_m;-}--static_assert(get_VKQ_stride(128, 1, 32) == 32, "Test failed.");-static_assert(get_VKQ_stride(128, 2, 32) == 64, "Test failed.");-static_assert(get_VKQ_stride(128, 4, 32) == 128, "Test failed.");-static_assert(get_VKQ_stride( 64, 1, 32) == 32, "Test failed.");-static_assert(get_VKQ_stride( 64, 2, 32) == 64, "Test failed.");-static_assert(get_VKQ_stride( 64, 4, 32) == 64, "Test failed.");-static_assert(get_VKQ_stride( 80, 1, 16) == 16, "Test failed.");-static_assert(get_VKQ_stride( 80, 2, 16) == 16, "Test failed.");-static_assert(get_VKQ_stride( 80, 4, 16) == 16, "Test failed.");--template <int D, int cols_per_block, typename KQ_acc_t>-void ggml_cuda_flash_attn_ext_wmma_f16_case(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {- const ggml_tensor * KQV = dst;-- constexpr int nwarps = 4;-- constexpr int frag_m = cols_per_block == 8 && D % 32 == 0 ? 32 : 16;- const int warp_size = ggml_cuda_info().devices[ggml_cuda_get_device()].warp_size;-- float logit_softcap;- memcpy(&logit_softcap, (const float *) KQV->op_params + 2, sizeof(float));-- fattn_kernel_t fattn_kernel;- if (logit_softcap == 0.0f) {- constexpr bool use_logit_softcap = false;- fattn_kernel = flash_attn_ext_f16<- D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), KQ_acc_t, use_logit_softcap>;- } else {- constexpr bool use_logit_softcap = true;- fattn_kernel = flash_attn_ext_f16<- D, cols_per_block, nwarps, get_VKQ_stride(D, nwarps, frag_m), KQ_acc_t, use_logit_softcap>;- }- launch_fattn<D, cols_per_block, 1>(ctx, dst, fattn_kernel, nwarps, 0, FATTN_KQ_STRIDE, true, true, false, warp_size);-}--void ggml_cuda_flash_attn_ext_wmma_f16(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {- const ggml_tensor * KQV = dst;- const ggml_tensor * Q = dst->src[0];-- const enum ggml_prec prec = ggml_flash_attn_ext_get_prec(KQV);- const int warp_size = ggml_cuda_info().devices[ctx.device].warp_size;-- if (prec != GGML_PREC_DEFAULT) {- if (Q->ne[1] <= 32 || Q->ne[0] > 128) {- constexpr int cols_per_block = 16;- switch (Q->ne[0]) {- case 64:- ggml_cuda_flash_attn_ext_wmma_f16_case< 64, cols_per_block, float>(ctx, dst);- break;- case 80:- ggml_cuda_flash_attn_ext_wmma_f16_case< 80, cols_per_block, float>(ctx, dst);- break;- case 96:- ggml_cuda_flash_attn_ext_wmma_f16_case< 96, cols_per_block, float>(ctx, dst);- break;- case 112:- ggml_cuda_flash_attn_ext_wmma_f16_case<112, cols_per_block, float>(ctx, dst);- break;- case 128:- ggml_cuda_flash_attn_ext_wmma_f16_case<128, cols_per_block, float>(ctx, dst);- break;- case 256:- ggml_cuda_flash_attn_ext_wmma_f16_case<256, cols_per_block, float>(ctx, dst);- break;- default:- GGML_ABORT("fatal error");- break;- }- } else {- constexpr int cols_per_block = 32;- switch (Q->ne[0]) {- case 64:- ggml_cuda_flash_attn_ext_wmma_f16_case< 64, cols_per_block, float>(ctx, dst);- break;- case 80:- ggml_cuda_flash_attn_ext_wmma_f16_case< 80, cols_per_block, float>(ctx, dst);- break;- case 96:- ggml_cuda_flash_attn_ext_wmma_f16_case< 96, cols_per_block, float>(ctx, dst);- break;- case 112:- ggml_cuda_flash_attn_ext_wmma_f16_case<112, cols_per_block, float>(ctx, dst);- break;- case 128:- ggml_cuda_flash_attn_ext_wmma_f16_case<128, cols_per_block, float>(ctx, dst);- break;- // case 256:- // ggml_cuda_flash_attn_ext_wmma_f16_case<256, cols_per_block, float>(ctx, dst);- // break;- default:- GGML_ABORT("fatal error");- break;- }- }- return;- }--#if !defined(GGML_USE_HIP)- if (Q->ne[1] <= 8 && Q->ne[0] % warp_size == 0) {- constexpr int cols_per_block = 8;- switch (Q->ne[0]) {- case 64:- ggml_cuda_flash_attn_ext_wmma_f16_case< 64, cols_per_block, half>(ctx, dst);- break;- case 96:- ggml_cuda_flash_attn_ext_wmma_f16_case< 96, cols_per_block, half>(ctx, dst);- break;- case 128:- ggml_cuda_flash_attn_ext_wmma_f16_case<128, cols_per_block, half>(ctx, dst);- break;- case 256:- ggml_cuda_flash_attn_ext_wmma_f16_case<256, cols_per_block, half>(ctx, dst);- break;- default:- GGML_ABORT("fatal error");- break;- }- return;- }-#endif // !defined(GGML_USE_HIP)-- if (Q->ne[1] <= 32) {- constexpr int cols_per_block = 16;- switch (Q->ne[0]) {- case 64:- ggml_cuda_flash_attn_ext_wmma_f16_case< 64, cols_per_block, half>(ctx, dst);- break;- case 80:- ggml_cuda_flash_attn_ext_wmma_f16_case< 80, cols_per_block, half>(ctx, dst);- break;- case 96:- ggml_cuda_flash_attn_ext_wmma_f16_case< 96, cols_per_block, half>(ctx, dst);- break;- case 112:- ggml_cuda_flash_attn_ext_wmma_f16_case<112, cols_per_block, half>(ctx, dst);- break;- case 128:- ggml_cuda_flash_attn_ext_wmma_f16_case<128, cols_per_block, half>(ctx, dst);- break;- case 256:- ggml_cuda_flash_attn_ext_wmma_f16_case<256, cols_per_block, half>(ctx, dst);- break;- default:- GGML_ABORT("fatal error");- break;- }- return;- }-- constexpr int cols_per_block = 32;- switch (Q->ne[0]) {- case 64:- ggml_cuda_flash_attn_ext_wmma_f16_case< 64, cols_per_block, half>(ctx, dst);- break;- case 80:- ggml_cuda_flash_attn_ext_wmma_f16_case< 80, cols_per_block, half>(ctx, dst);- break;- case 96:- ggml_cuda_flash_attn_ext_wmma_f16_case< 96, cols_per_block, half>(ctx, dst);- break;- case 112:- ggml_cuda_flash_attn_ext_wmma_f16_case<112, cols_per_block, half>(ctx, dst);- break;- case 128:- ggml_cuda_flash_attn_ext_wmma_f16_case<128, cols_per_block, half>(ctx, dst);- break;- case 256:- ggml_cuda_flash_attn_ext_wmma_f16_case<256, cols_per_block, half>(ctx, dst);- break;- default:- GGML_ABORT("fatal error");- break;- }-}diff --git a/ggml/src/ggml-cuda/fattn-wmma-f16.cuh b/ggml/src/ggml-cuda/fattn-wmma-f16.cuhdeleted file mode 100644index aaf711a618..0000000000--- a/ggml/src/ggml-cuda/fattn-wmma-f16.cuh+++ /dev/null@@ -1,51 +0,0 @@-#pragma once--#include "common.cuh"--#if defined(GGML_USE_MUSA)-#define GGML_USE_WMMA_FATTN-#endif // defined(GGML_USE_MUSA)--#if defined(GGML_HIP_ROCWMMA_FATTN)-#if defined(CDNA) && (ROCWMMA_VERSION_MAJOR < 2 || ROCWMMA_VERSION_MINOR > 0 || ROCWMMA_VERSION_PATCH > 0)-#define GGML_USE_WMMA_FATTN-#elif defined(CDNA)-#warning "rocwmma fattn on CDNA is broken on rocwmma v2.0.0, expect degraded performance"-#endif // defined(CDNA) && (ROCWMMA_VERSION_MAJOR < 2 || ROCWMMA_VERSION_MINOR > 0 || ROCWMMA_VERSION_PATCH > 0)-#if defined(RDNA3)-#define GGML_USE_WMMA_FATTN-#endif // defined(RDNA3)-#if defined(RDNA4) && ROCWMMA_VERSION_MAJOR > 1-#define GGML_USE_WMMA_FATTN-#elif defined(RDNA4)-#warning "rocwmma fattn is not supported on RDNA4 on rocwmma < v2.0.0, expect degraded performance"-#endif // defined(RDNA4) && ROCWMMA_VERSION_MAJOR > 1-#endif // defined(GGML_HIP_ROCWMMA_FATTN)--// WMMA flash attention requires FP16 matrix instructions to be available for ggml code.-static bool ggml_cuda_should_use_wmma_fattn(const int cc) {-#if defined(GGML_USE_HIP) && !defined(GGML_HIP_ROCWMMA_FATTN)- return false;-#else- if ((GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_VOLTA) ||- GGML_CUDA_CC_IS_RDNA3(cc) || GGML_CUDA_CC_IS_MTHREADS(cc)) {- return true;- } else if (GGML_CUDA_CC_IS_CDNA(cc)){-#if defined(GGML_HIP_ROCWMMA_FATTN) && (ROCWMMA_VERSION_MAJOR < 2 || ROCWMMA_VERSION_MINOR > 0 || ROCWMMA_VERSION_PATCH > 0)- return true;-#else- return false;-#endif // defined(GGML_HIP_ROCWMMA_FATTN) (ROCWMMA_VERSION_MAJOR < 2 || ROCWMMA_VERSION_MINOR > 0 || ROCWMMA_VERSION_PATCH > 0)- } else if (GGML_CUDA_CC_IS_RDNA4(cc)) {-#if defined(GGML_HIP_ROCWMMA_FATTN) && ROCWMMA_VERSION_MAJOR > 1- return true;-#else- return false;-#endif // defined(GGML_HIP_ROCWMMA_FATTN) && ROCWMMA_VERSION_MAJOR > 1- } else {- return false;- }-#endif // defined(GGML_USE_HIP) && !defined(GGML_HIP_ROCWMMA_FATTN)-}--void ggml_cuda_flash_attn_ext_wmma_f16(ggml_backend_cuda_context & ctx, ggml_tensor * dst);diff --git a/ggml/src/ggml-cuda/fattn.cu b/ggml/src/ggml-cuda/fattn.cuindex 00ffacf299..ab7a3b297c 100644--- a/ggml/src/ggml-cuda/fattn.cu+++ b/ggml/src/ggml-cuda/fattn.cu@@ -3,7 +3,6 @@#include "fattn-mma-f16.cuh"#include "fattn-tile.cuh"#include "fattn-vec.cuh"-#include "fattn-wmma-f16.cuh"#include "fattn.cuh"template <int DKQ, int DV, int ncols2>@@ -330,11 +329,10 @@ static void ggml_cuda_flash_attn_ext_vec(ggml_backend_cuda_context & ctx, ggml_t// Best FlashAttention kernel for a specific GPU:enum best_fattn_kernel {- BEST_FATTN_KERNEL_NONE = 0,- BEST_FATTN_KERNEL_TILE = 200,- BEST_FATTN_KERNEL_VEC = 100,- BEST_FATTN_KERNEL_WMMA_F16 = 300,- BEST_FATTN_KERNEL_MMA_F16 = 400,+ BEST_FATTN_KERNEL_NONE = 0,+ BEST_FATTN_KERNEL_TILE = 200,+ BEST_FATTN_KERNEL_VEC = 100,+ BEST_FATTN_KERNEL_MMA_F16 = 400,};static bool ggml_cuda_fattn_kv_type_supported(ggml_type type) {@@ -500,14 +498,6 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, constreturn BEST_FATTN_KERNEL_MMA_F16;}- // Use the WMMA kernel if possible:- if (ggml_cuda_should_use_wmma_fattn(cc) && K->ne[1] % FATTN_KQ_STRIDE == 0 && Q->ne[0] != 40 && Q->ne[0] != 72 && Q->ne[0] != 192 && Q->ne[0] != 512 && Q->ne[0] != 576) {- if (can_use_vector_kernel && Q->ne[1] <= 2) {- return BEST_FATTN_KERNEL_VEC;- }- return BEST_FATTN_KERNEL_WMMA_F16;- }-// AMD MFMA needs a certain minimum batch size to outscale the tile kernel for large head sizes.if ((amd_mfma_available(cc) && Q->ne[0] <= 256) && Q->ne[0] != 40 && Q->ne[0] != 72) {if ((Q->ne[0] <= 64 && Q->ne[1] * gqa_ratio_eff > 8)) {@@ -559,7 +549,6 @@ size_t ggml_cuda_flash_attn_ext_get_alloc_size(int device, const ggml_tensor * dswitch (kernel) {case BEST_FATTN_KERNEL_TILE:- case BEST_FATTN_KERNEL_WMMA_F16:case BEST_FATTN_KERNEL_MMA_F16:need_f16_K = true;need_f16_V = true;@@ -589,9 +578,6 @@ void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dstcase BEST_FATTN_KERNEL_VEC:ggml_cuda_flash_attn_ext_vec(ctx, dst);break;- case BEST_FATTN_KERNEL_WMMA_F16:- ggml_cuda_flash_attn_ext_wmma_f16(ctx, dst);- break;case BEST_FATTN_KERNEL_MMA_F16:ggml_cuda_flash_attn_ext_mma_f16(ctx, dst);break;diff --git a/ggml/src/ggml-cuda/vendors/hip.h b/ggml/src/ggml-cuda/vendors/hip.hindex d01f1533ab..9aa558f3f4 100644--- a/ggml/src/ggml-cuda/vendors/hip.h+++ b/ggml/src/ggml-cuda/vendors/hip.h@@ -6,10 +6,6 @@#include <hip/hip_fp16.h>#include <hip/hip_bf16.h>-#if defined(GGML_HIP_ROCWMMA_FATTN)-#include <rocwmma/rocwmma-version.hpp>-#endif // defined(GGML_HIP_ROCWMMA_FATTN)-#ifdef GGML_USE_NCCL#include <rccl/rccl.h>#endif // GGML_USE_NCCLdiff --git a/ggml/src/ggml-hip/CMakeLists.txt b/ggml/src/ggml-hip/CMakeLists.txtindex 7121193f1c..5351dcae12 100644--- a/ggml/src/ggml-hip/CMakeLists.txt+++ b/ggml/src/ggml-hip/CMakeLists.txt@@ -114,10 +114,6 @@ if (GGML_HIP_NO_VMM)add_compile_definitions(GGML_HIP_NO_VMM)endif()-if (GGML_HIP_ROCWMMA_FATTN)- add_compile_definitions(GGML_HIP_ROCWMMA_FATTN)-endif()-if (NOT GGML_HIP_MMQ_MFMA)add_compile_definitions(GGML_HIP_NO_MMQ_MFMA)endif()