LLM inference in C/C++
C++ ggml
Fixing the vulkan build issue of legacy GLSLC version that has no cooperativeMatrix API support (https://github.com/ggml-org/llama.cpp/issues/29373) (#29409)
sliu39e85e15cf6d81
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cppindex bd416a4e06..ce40baec2d 100644--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp@@ -3170,6 +3170,7 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {ggml_vk_create_pipeline(device, device->pipeline_matmul_split_k_reduce, "split_k_reduce", split_k_reduce_len, split_k_reduce_data, "main", 2, 2 * sizeof(uint32_t), {256 * 4, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_flash_attn_split_k_reduce, "fa_split_k_reduce", fa_split_k_reduce_len, fa_split_k_reduce_data, "main", 3, sizeof(vk_op_flash_attn_split_k_reduce_push_constants), {1, device->subgroup_size, 1}, {device->subgroup_size}, 1, true);+#if defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT)if (device->vendor_id == VK_VENDOR_ID_INTEL && (device->architecture == INTEL_XE2 || (device->architecture == INTEL_XE1 && device->coopmat_support && device->uma))) {auto upper_power_of_2 = [&](uint32_t in) {GGML_ASSERT(in != 0);@@ -3209,6 +3210,7 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {ggml_vk_create_pipeline(device, pipelines.second, "xe_fa_decode_ph2", fa_decode_ph2_cm1_len, fa_decode_ph2_cm1_data, "main", 5, sizeof(vk_fa_xe_opt_push_constants), { 1, 1, 1 }, { group_sz_ph2, gqa_ratio, head_dim_pv, out_per_wg_ph2, xe_native_sub_group_size, split_p_per_iter_ph2, split_p_chunk, out_dim_per_wg }, 1, false, true, xe_native_sub_group_size);}}+#endiffor (auto &it : device->pipeline_fa_mask_opt) {auto BrBc = it.first;@@ -8219,10 +8221,11 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, constsplit_k = CEIL_DIV(KV, split_kv);xe_fa_opt = xe_fa_supported_platform && xe_fa_supported_usage && xe_fa_supported_dtype;if (xe_fa_opt) {- std::lock_guard<std::mutex> guard(ctx->device->compile_mutex);const uint32_t split_p_size = 32;const size_t max_dim = (nek1 + split_p_size - 1) / split_p_size;const size_t p_dim = max_dim * split_p_size;+#if defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT)+ std::lock_guard<std::mutex> guard(ctx->device->compile_mutex);auto& pipelines = ctx->device->pipeline_xe_fa_decode_dual_phases;auto it = pipelines.find({ (uint32_t)neq0, (uint32_t)nev0, qk_ratio, (uint32_t)neq1 });if (it != pipelines.end()) {@@ -8230,18 +8233,23 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, const} else {pipelines[{(uint32_t)neq0, (uint32_t)nev0, qk_ratio, (uint32_t)neq1}] = xe_fa_pipeline_dual_phases = std::make_pair(std::make_shared<vk_pipeline_struct>(), std::make_shared<vk_pipeline_struct>());}+#endif+ if (xe_fa_pipeline_dual_phases.first == nullptr || xe_fa_pipeline_dual_phases.second == nullptr) {+ xe_fa_opt = false;+ fa_copy_qstate = false;+ } else {+ size_p = neq1 * neq2 * p_dim * neq3 * sizeof(ggml_fp16_t);+ size_group_max = neq1 * neq2 * max_dim * neq3 * sizeof(float);+ size_t temp_size = ggml_nelements(q) * sizeof(ggml_fp16_t) + size_p + size_group_max;+ fa_copy_qstate = true;+ if (ctx->prealloc_size_x < temp_size) {+ ctx->prealloc_size_x = temp_size;+ ggml_vk_preallocate_buffers(ctx, subctx);+ }- size_p = neq1 * neq2 * p_dim * neq3 * sizeof(ggml_fp16_t);- size_group_max = neq1 * neq2 * max_dim * neq3 * sizeof(float);- size_t temp_size = ggml_nelements(q) * sizeof(ggml_fp16_t) + size_p + size_group_max;- fa_copy_qstate = true;- if (ctx->prealloc_size_x < temp_size) {- ctx->prealloc_size_x = temp_size;- ggml_vk_preallocate_buffers(ctx, subctx);- }-- if (ctx->prealloc_x_need_sync) {- ggml_vk_sync_buffers(ctx, subctx);+ if (ctx->prealloc_x_need_sync) {+ ggml_vk_sync_buffers(ctx, subctx);+ }}}}diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cppindex cde5d36dce..e7e303e50c 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp+++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp@@ -929,8 +929,10 @@ void process_shaders() {string_to_spv("fa_mask_opt", "flash_attn_mask_opt.comp", {});+#if defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT)string_to_spv("fa_decode_ph1", "flash_attn_decode_phase_1.comp", {}, true, true, false, false);string_to_spv("fa_decode_ph2", "flash_attn_decode_phase_2.comp", {}, true, true, false, false);+#endifstring_to_spv("fa_sparse_compact", "flash_attn_sparse_compact.comp", {});string_to_spv("fa_sparse_compact_subgroup", "flash_attn_sparse_compact.comp", {{"USE_SUBGROUPS", "1"}});