LLM inference in C/C++
C++ ggml
graph: pad the KQ bias of the no-cache attention (#30446)
Pascalace20f679e30
diff --git a/src/llama-graph.cpp b/src/llama-graph.cppindex 9039e47fd8..f69fe63f8f 100644--- a/src/llama-graph.cpp+++ b/src/llama-graph.cpp@@ -3056,12 +3056,16 @@ ggml_tensor * llm_graph_context::build_attn(ggml_tensor * k = k_cur;ggml_tensor * v = v_cur;- // the KV values are padded with zeros up to the KV dimension of the mask+ // the KV values and the KQ bias are padded with zeros up to the KV dimension of the maskconst int64_t n_kv_pad = kq_mask->ne[0] - k->ne[2];k = ggml_pad(ctx0, k, 0, 0, n_kv_pad, 0);v = ggml_pad(ctx0, v, 0, 0, n_kv_pad, 0);+ if (kq_b) {+ kq_b = ggml_pad(ctx0, kq_b, n_kv_pad, 0, 0, 0);+ }+ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, 0, kq_scale, il);cb(cur, "kqv_out", il);