LLM inference in C/C++
C++ ggml
HIP/CUDA: Fix reduction in mmf getting tile widths not devisiable by 64 (#30836)
uvosb68449857989
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuindex d9b18cc08d..f6ff9f893a 100644--- a/ggml/src/ggml-cuda/ggml-cuda.cu+++ b/ggml/src/ggml-cuda/ggml-cuda.cu@@ -2033,6 +2033,8 @@ static bool ggml_cuda_mul_mat_id_needs_sync(const ggml_tensor * dst, const int cconst ggml_tensor * src0 = dst->src[0];const ggml_tensor * src1 = dst->src[1];+ const int warp_size = ggml_cuda_info().devices[ggml_cuda_get_device()].warp_size;+if (src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {return true;}@@ -2051,7 +2053,7 @@ static bool ggml_cuda_mul_mat_id_needs_sync(const ggml_tensor * dst, const int creturn false;}- if (ggml_cuda_should_use_mmf(src0->type, cc, WARP_SIZE, src0->ne, src0->nb, src1->ne[2], /*mul_mat_id=*/true)) {+ if (ggml_cuda_should_use_mmf(src0->type, cc, warp_size, src0->ne, src0->nb, src1->ne[2], /*mul_mat_id=*/true)) {return false;}@@ -2069,6 +2071,7 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor *GGML_TENSOR_BINARY_OP_LOCALSconst int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;+ const int warp_size = ggml_cuda_info().devices[ggml_cuda_get_device()].warp_size;// [TAG_MUL_MAT_ID_CUDA_GRAPHS]if (src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {@@ -2093,7 +2096,7 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor *return;}- if (ggml_cuda_should_use_mmf(src0->type, cc, WARP_SIZE, src0->ne, src0->nb, src1->ne[2], /*mul_mat_id=*/true)) {+ if (ggml_cuda_should_use_mmf(src0->type, cc, warp_size, src0->ne, src0->nb, src1->ne[2], /*mul_mat_id=*/true)) {ggml_cuda_mul_mat_f(ctx, src0, src1, ids, dst);return;}