LLM inference in C/C++
C++ ggml
ggml-cuda: fattn do not check oob if k_VKQ_max = (k_VKQ_max - nbatch_fa) (#30572)
thelittlefireman5aeac37a35b4
diff --git a/ggml/src/ggml-cuda/fattn-tile.cuh b/ggml/src/ggml-cuda/fattn-tile.cuhindex b5ea915e13..bd4a7710f3 100644--- a/ggml/src/ggml-cuda/fattn-tile.cuh+++ b/ggml/src/ggml-cuda/fattn-tile.cuh@@ -955,7 +955,7 @@ static __global__ void flash_attn_tile(if (ncols2 == 1) {// Branch with out-of-bounds checks.int k_VKQ_0 = blockIdx.y*nbatch_fa;- while (k_VKQ_0 < k_VKQ_max - nbatch_fa) {+ while (k_VKQ_0 <= k_VKQ_max - nbatch_fa) {constexpr bool oob_check = false;flash_attn_tile_iter<warp_size, nwarps, ncols1, ncols2, DKQ, DV, nbatch_fa, nbatch_K, use_logit_softcap, oob_check>(Q_tmp, K_h2, V_h2, maskh, ne01, logit_softcap, slope, KQ, KV_tmp,