LLM inference in C/C++
C++ ggml
vulkan: fix stale prealloc_y reuse across flash attention and soft_max (#29591)
François-Xavier Gsell806eee9841de
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cppindex 0588a00403..735ae3b947 100644--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp@@ -8376,6 +8376,11 @@ void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx, constvk_subbuffer sinks_buf = sinks ? ggml_vk_tensor_subbuffer(ctx, sinks) : q_buf;vk_subbuffer mask_opt_buf = use_mask_opt ? ggml_vk_subbuffer(ctx, ctx->prealloc_y, 0) : q_buf;vk_subbuffer sparse_buf = use_sparse ? ggml_vk_subbuffer(ctx, ctx->prealloc_y, 0) : q_buf;+ if (use_mask_opt || use_sparse) {+ // the mask opt bits and the sparse index list overwrite a matmul input converted into prealloc_y+ ctx->prealloc_y_last_pipeline_used = nullptr;+ ctx->prealloc_y_last_tensor_used = nullptr;+ }if (use_dequant_kv) {const uint64_t fp = sizeof(ggml_fp16_t);@@ -11059,6 +11064,10 @@ void ggml_vk_soft_max(ggml_backend_vk_context * ctx, vk_context& subctx, const gvk_subbuffer buf_x = { ctx->prealloc_x, 0, tmp_size };vk_subbuffer buf_y = { ctx->prealloc_y, 0, tmp_size };+ // the partial results overwrite a matmul input converted into prealloc_y+ ctx->prealloc_y_last_pipeline_used = nullptr;+ ctx->prealloc_y_last_tensor_used = nullptr;+std::array<uint32_t, 3> elements = { num_wgs, nrows_x, 1 };vk_pipeline pipeline1 = src1 && src1->type == GGML_TYPE_F16 ? ctx->device->pipeline_soft_max_large1_f32_f16 : ctx->device->pipeline_soft_max_large1_f32;