Skip to content

LLM inference in C/C++

C++ ggml

vulkan: fix FA dequant path engagement (#28190)

Nathan Wilsonc7bda030e7fa
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
index 285cd65253..a04a6b27a8 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp
+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
@@ -10966,7 +10966,7 @@ static void ggml_vk_flash_attn(ggml_backend_vk_context * ctx, vk_context& subctx
return t->nb[0] == ggml_type_size(t->type) &&
t->nb[2] == ggml_row_size(t->type, t->ne[0]) &&
t->nb[1] == t->nb[2] * t->ne[2] &&
- t->nb[3] == t->nb[1] * t->ne[1];
+ (t->ne[3] == 1 || t->nb[3] == t->nb[1] * t->ne[1]);
};
const bool k_quant = k->type != GGML_TYPE_F16 && k->type != GGML_TYPE_BF16 && k->type != GGML_TYPE_F32;
const bool v_quant = v->type != GGML_TYPE_F16 && v->type != GGML_TYPE_BF16 && v->type != GGML_TYPE_F32;