LLM inference in C/C++
C++ ggml
vulkan: Convert FILL to distribute workgroups in 2D to avoid exceeding maxComputeWorkGroupCount (#28592)
Masato Nakasaka22397c31a00e
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cppindex b3a7cb6ab6..9eae8dab9c 100644--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp@@ -13748,9 +13748,11 @@ static void ggml_vk_arange(ggml_backend_vk_context * ctx, vk_context& subctx, ggstatic void ggml_vk_fill(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {VK_LOG_DEBUG("ggml_vk_fill(dst=" << dst << ", ne=" << ggml_nelements(dst) << ")");+ const uint64_t n = ggml_nelements(dst);+ GGML_ASSERT(n > 0);vk_op_push_constants pc = {- (uint32_t)ggml_nelements(dst),+ (uint32_t)n,1,ggml_get_op_params_f32(dst, 0),0.0f,@@ -13760,11 +13762,16 @@ static void ggml_vk_fill(ggml_backend_vk_context * ctx, vk_context& subctx, ggmlvk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, nullptr, nullptr, nullptr, dst, GGML_OP_FILL);GGML_ASSERT(pipeline != nullptr);+ // Split the task distribution to 2D to avoid exceeding maxComputeWorkGroupCount+ const uint32_t total_wg = CEIL_DIV(n, pipeline->wg_denoms[0]);+ const uint32_t wg_x = std::min(total_wg, ctx->device->properties.limits.maxComputeWorkGroupCount[0]);+ const uint32_t wg_y = CEIL_DIV(total_wg, wg_x);+ GGML_ASSERT(wg_y <= ctx->device->properties.limits.maxComputeWorkGroupCount[1]);+ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1);vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst, false);- std::array<uint32_t, 3> elements = { (uint32_t)ggml_nelements(dst), 1, 1 };-+ std::array<uint32_t, 3> elements = { wg_x * pipeline->wg_denoms[0], wg_y * pipeline->wg_denoms[1], 1 };ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { dst_buf }, pc, elements);}diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/fill.comp b/ggml/src/ggml-vulkan/vulkan-shaders/fill.compindex a56be76c61..b5cc333220 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/fill.comp+++ b/ggml/src/ggml-vulkan/vulkan-shaders/fill.comp@@ -8,7 +8,9 @@ layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;layout (binding = 0) writeonly buffer D {D_TYPE data_d[];};void main() {- const uint i = gl_GlobalInvocationID.x;+ // 2D grid flattening: each x workgroup covers gl_WorkGroupSize.x elements,+ // each y workgroup covers gl_NumWorkGroups.x * gl_WorkGroupSize.x elements.+ const uint i = (gl_GlobalInvocationID.y * gl_NumWorkGroups.x * gl_WorkGroupSize.x) + gl_GlobalInvocationID.x;if (i >= p.KX) {return;