LLM inference in C/C++
C++ ggml
cuda: cap MMQ Stream-K blocks at available work (#30553)
Joel4c1004c49221
diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuhindex 33449f9058..e008a34c08 100644--- a/ggml/src/ggml-cuda/mmq.cuh+++ b/ggml/src/ggml-cuda/mmq.cuh@@ -1441,10 +1441,12 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & aconst int ntiles_dst = ntx * nty * ntzw;const int tiles_nwaves = (ntiles_dst + nsm - 1) / nsm;const int tiles_efficiency_percent = 100 * ntiles_dst / (nsm*tiles_nwaves);- const dim3 block_nums_stream_k(GGML_CUDA_CC_IS_NVIDIA(cc) && tiles_efficiency_percent >= 90 ? ntiles_dst : nsm, 1, 1);GGML_ASSERT(ntiles_dst * blocks_per_ne00_fd.z < (1 << 30)); // Assert that variable kbc will not overflow.+ const int nwork = ntiles_dst * ((args.ncols_x + config.K_vram - 1) / config.K_vram); // Maximum number of CUDA blocks that can do useful work.+ const dim3 block_nums_stream_k(GGML_CUDA_CC_IS_NVIDIA(cc) && tiles_efficiency_percent >= 90 ? ntiles_dst : std::min(nsm, nwork), 1, 1);+const bool fixup_needed = ntiles_dst % block_nums_stream_k.x != 0;ggml_cuda_pool & pool = ctx.pool(id);