LLM inference in C/C++
C++ ggml
HIP: fattn-mma: use fp32 accumulation on MFMA devices (#28576)
uvosbfdc32183d57
diff --git a/ggml/src/ggml-cuda/fattn-mma-f16.cuh b/ggml/src/ggml-cuda/fattn-mma-f16.cuhindex 578f6cf79c..a290655776 100644--- a/ggml/src/ggml-cuda/fattn-mma-f16.cuh+++ b/ggml/src/ggml-cuda/fattn-mma-f16.cuh@@ -181,7 +181,7 @@ static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_coGGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 8, 128, 1, 64, 32, 32, 32, 1, true);GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 16, 256, 2, 64, 32, 32, 32, 1, true);GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 32, 256, 2, 64, 32, 32, 32, 1, true);- GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 64, 256, 4, 64, 32, 32, 32, 1, true);+ GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64, 64, 64, 256, 3, 64, 32, 32, 32, 1, true);GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 8, 256, 2, 64, 40, 40, 40, 1, true);GGML_CUDA_FATTN_MMA_CONFIG_CASE( 80, 80, 16, 256, 2, 64, 40, 40, 40, 1, true);@@ -1141,7 +1141,7 @@ template<int DV, int ncols> struct mma_tile_sizes {using T_C_KQ = tile<16, 16, float>; // column-majorusing T_A_VKQ = tile<16, 8, half2>; // row-majorusing T_B_VKQ = tile<16, 8, half2>; // column-major- using T_C_VKQ = tile<16, 8, half2>; // column-major+ using T_C_VKQ = tile<16, 16, float>; // column-major};#else // Voltatemplate<int DV, int ncols> struct mma_tile_sizes {@@ -1227,7 +1227,9 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(T_C_VKQ VKQ_C[cols_per_warp == 8 ? DV/T_C_VKQ::I : DV/(2*T_C_VKQ::J)];#elif defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)T_C_VKQ VKQ_C[DV % 32 != 0 ? DV/T_C_VKQ::J : DV/(2*T_C_VKQ::J)];-#elif defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)+#elif defined(AMD_MFMA_AVAILABLE)+ T_C_VKQ VKQ_C[ DV/T_C_VKQ::J];+#elif defined(AMD_WMMA_AVAILABLE)T_C_VKQ VKQ_C[ DV/(2*T_C_VKQ::J)];#else // VoltaT_C_VKQ VKQ_C[ DV/(2*T_C_VKQ::J)];