LLM inference in C/C++
C++ ggml
vulkan: use 4 rows for NVIDIA MUL_MAT_ID MMVQ (#29274)
Amadeus dyw404f557b5b14
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cppindex 42e407ebfe..907ac82ce7 100644--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp@@ -2892,8 +2892,14 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {// RDNA3/4: above four columns, static 4 rows for all types bench faster than the defaultconst bool is_rdna3_or_4 = device->vendor_id == VK_VENDOR_ID_AMD && (device->architecture == AMD_RDNA3 || device->architecture == AMD_RDNA4);auto const &rm_int_n = [&](uint32_t rows, uint32_t i) { return (is_rdna3_or_4 && i >= 4) ? 4u : rows; };- // RDNA3/4: Static 4 rows for all types bench faster than the default- auto const &rm_id = [&](uint32_t rows) { return is_rdna3_or_4 ? 4u : rows; };+ // RDNA3/4 and NVIDIA except pre-Turing: use 4 rows for MUL_MAT_ID MMVQ.+ auto const &rm_id = [&](uint32_t rows) {+ if (device->vendor_id == VK_VENDOR_ID_NVIDIA &&+ device->architecture != vk_device_architecture::NVIDIA_PRE_TURING) {+ return 4u;+ }+ return is_rdna3_or_4 ? 4u : rows;+ };uint32_t rm_iq = 2 * rm_kq;const bool use_subgroups = device->subgroup_arithmetic;