LLM inference in C/C++
C++ ggml
CUDA: Optimize accumulation in mmq for NVFP4 type (#29857)
Konrad Moren8b2fbaf32cad
diff --git a/ggml/src/ggml-cuda/mmq-vec-dot.cuh b/ggml/src/ggml-cuda/mmq-vec-dot.cuhindex 4ca6542d80..b39e4d579a 100644--- a/ggml/src/ggml-cuda/mmq-vec-dot.cuh+++ b/ggml/src/ggml-cuda/mmq-vec-dot.cuh@@ -1225,14 +1225,11 @@ template <ggml_type type, int J, bool fallback> static __device__ __forceinline_#pragma unrollfor (int n = 0; n < ntx; ++n) {+ // accumulate in place into the output sum array+ tile_C & C = *reinterpret_cast<tile_C *>(sum + (j0 / tile_C::J + n) * tile_C::ne);#pragma unrollfor (int frag = 0; frag < nfrags; ++frag) {- tile_C C = {};mma_block_scaled_fp4<type>(C, A[n][frag], B[frag], scaleA[n][frag], scaleB[frag]);-#pragma unroll- for (int l = 0; l < tile_C::ne; ++l) {- sum[(j0 / tile_C::J + n) * tile_C::ne + l] += C.x[l];- }}}}