LLM inference in C/C++
C++ ggml
opencl: handle OOB write in noshuffle GEMV kernels (odd ne01) (#25640)
Hongqiang Wang00fa7cb284cb
diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.clindex 9386bf25a6..1f832cb253 100644--- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl@@ -296,7 +296,12 @@ kernel void kernel_gemv_noshuffle_iq4_nl_f32(// 2 outputs per fiber in wave 0if (groupId == 0) {dst = (global float*)((global char*)dst + offsetd);- vstore2(totalSum, 0, &(dst[gid * 2]));+ // Guard the two output rows. The x-grid is padded to CEIL_DIV(ne01/2,64)*64,+ // so when ne01 is not a multiple of 128 the tail row-pairs run past row ne01+ // and would overrun dst into the adjacent tensor. No-op / byte-identical when+ // ne01 % 128 == 0 (M/2 already a multiple of 64 -> no padding).+ if (gid * 2 + 0 < M) dst[gid * 2 + 0] = totalSum.s0;+ if (gid * 2 + 1 < M) dst[gid * 2 + 1] = totalSum.s1;}}diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.clindex e83c5d0689..9efede2941 100644--- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl@@ -116,6 +116,10 @@ __kernel void kernel_gemv_noshuffle_q1_0_f32(if (groupId == 0) {dst = (global float*)((global char*)dst + offsetd);- dst[gid] = totalSum;+ // Guard the output row. The x-grid is padded to CEIL_DIV(M,wavesize)*wavesize,+ // so when ne01 is not a multiple of the wave size the tail work-items run past+ // row ne01 and would overrun dst into the adjacent tensor. No-op / byte-identical+ // when ne01 is wave-aligned (no padding).+ if (gid < M) dst[gid] = totalSum;}}diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.clindex 1068320691..8de0de1cc3 100644--- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl@@ -268,7 +268,12 @@ __kernel void kernel_gemv_noshuffle_q4_0_f32(// 2 outputs per fiber in wave 0if (groupId == 0) {dst = (global float*)((global char*)dst + offsetd);- vstore2(totalSum, 0, &(dst[gid * 2]));+ // Guard the two output rows. The x-grid is padded to CEIL_DIV(ne01/2,64)*64,+ // so when ne01 is not a multiple of 128 the tail row-pairs run past row ne01+ // and would overrun dst into the adjacent tensor. No-op / byte-identical when+ // ne01 % 128 == 0 (M/2 already a multiple of 64 -> no padding).+ if (gid * 2 + 0 < M) dst[gid * 2 + 0] = totalSum.s0;+ if (gid * 2 + 1 < M) dst[gid * 2 + 1] = totalSum.s1;}}diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.clindex 571a375da7..0dca20f71f 100644--- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl@@ -262,7 +262,11 @@ __kernel void kernel_gemv_noshuffle_q4_0_f32(// 2 outputs per fiber in wave 0if (groupId == 0) {dst = (global float*)((global char*)dst + offsetd);- vstore2(totalSum, 0, &(dst[gid * 2]));+ // Guard the two output rows against the padded x-grid tail overrunning dst.+ // The current shape specializations are all ne01 % 128 == 0 (no padding), so+ // this is a no-op / byte-identical today; keep it in lockstep with the base kernel.+ if (gid * 2 + 0 < ne01) dst[gid * 2 + 0] = totalSum.s0;+ if (gid * 2 + 1 < ne01) dst[gid * 2 + 1] = totalSum.s1;}}diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.clindex fdc1472454..5fa3127806 100644--- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl@@ -277,7 +277,12 @@ kernel void kernel_gemv_noshuffle_q4_1_f32(// 2 outputs per fiber in wave 0if (groupId == 0) {dst = (global float*)((global char*)dst + offsetd);- vstore2(totalSum, 0, &(dst[gid * 2]));+ // Guard the two output rows. The x-grid is padded to CEIL_DIV(ne01/2,64)*64,+ // so when ne01 is not a multiple of 128 the tail row-pairs run past row ne01+ // and would overrun dst into the adjacent tensor. No-op / byte-identical when+ // ne01 % 128 == 0 (M/2 already a multiple of 64 -> no padding).+ if (gid * 2 + 0 < M) dst[gid * 2 + 0] = totalSum.s0;+ if (gid * 2 + 1 < M) dst[gid * 2 + 1] = totalSum.s1;}}diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.clindex dd1e2b55c0..2eb20e2f76 100644--- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl@@ -312,7 +312,12 @@ kernel void kernel_gemv_noshuffle_q4_k_f32(// 2 outputs per fiber in wave 0if (groupId == 0) {dst = (global float*)((global char*)dst + offsetd);- vstore2(totalSum, 0, &(dst[gid * 2]));+ // Guard the two output rows. The x-grid is padded to CEIL_DIV(ne01/2,64)*64,+ // so when ne01 is not a multiple of 128 the tail row-pairs run past row ne01+ // and would overrun dst into the adjacent tensor. No-op / byte-identical when+ // ne01 % 128 == 0 (M/2 already a multiple of 64 -> no padding).+ if (gid * 2 + 0 < M) dst[gid * 2 + 0] = totalSum.s0;+ if (gid * 2 + 1 < M) dst[gid * 2 + 1] = totalSum.s1;}}diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.clindex c228f717a9..7dbf5a3bbb 100644--- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl@@ -285,7 +285,12 @@ __kernel void kernel_gemv_noshuffle_q5_0_f32(// 2 outputs per fiber in wave 0if (groupId == 0) {dst = (global float*)((global char*)dst + offsetd);- vstore2(totalSum, 0, &(dst[gid * 2]));+ // Guard the two output rows. The x-grid is padded to CEIL_DIV(ne01/2,64)*64,+ // so when ne01 is not a multiple of 128 the tail row-pairs run past row ne01+ // and would overrun dst into the adjacent tensor. No-op / byte-identical when+ // ne01 % 128 == 0 (M/2 already a multiple of 64 -> no padding).+ if (gid * 2 + 0 < M) dst[gid * 2 + 0] = totalSum.s0;+ if (gid * 2 + 1 < M) dst[gid * 2 + 1] = totalSum.s1;}}diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.clindex daf1308ea4..ba0e2a7115 100644--- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl@@ -288,7 +288,12 @@ __kernel void kernel_gemv_noshuffle_q5_1_f32(// 2 outputs per fiber in wave 0if (groupId == 0) {dst = (global float*)((global char*)dst + offsetd);- vstore2(totalSum, 0, &(dst[gid * 2]));+ // Guard the two output rows. The x-grid is padded to CEIL_DIV(ne01/2,64)*64,+ // so when ne01 is not a multiple of 128 the tail row-pairs run past row ne01+ // and would overrun dst into the adjacent tensor. No-op / byte-identical when+ // ne01 % 128 == 0 (M/2 already a multiple of 64 -> no padding).+ if (gid * 2 + 0 < M) dst[gid * 2 + 0] = totalSum.s0;+ if (gid * 2 + 1 < M) dst[gid * 2 + 1] = totalSum.s1;}}diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.clindex c40db16663..446f465338 100644--- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl@@ -321,6 +321,11 @@ kernel void kernel_gemv_noshuffle_q5_k_f32(// 2 outputs per fiber in wave 0if (groupId == 0) {dst = (global float*)((global char*)dst + offsetd);- vstore2(totalSum, 0, &(dst[gid * 2]));+ // Guard the two output rows. The x-grid is padded to CEIL_DIV(ne01/2,64)*64,+ // so when ne01 is not a multiple of 128 the tail row-pairs run past row ne01+ // and would overrun dst into the adjacent tensor. No-op / byte-identical when+ // ne01 % 128 == 0 (M/2 already a multiple of 64 -> no padding).+ if (gid * 2 + 0 < M) dst[gid * 2 + 0] = totalSum.s0;+ if (gid * 2 + 1 < M) dst[gid * 2 + 1] = totalSum.s1;}}diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.clindex 6f89cf968b..51682ecebb 100644--- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl@@ -288,6 +288,11 @@ kernel void kernel_gemv_noshuffle_q6_K_f32(if (grp == 0) {dst = (global float*)((global char*)dst + offsetd);- vstore2(total_sum, 0, &(dst[gid * 2]));+ // Guard the two output rows. The x-grid is padded to CEIL_DIV(ne01/2,64)*64,+ // so when ne01 is not a multiple of 128 the tail row-pairs run past row ne01+ // and would overrun dst into the adjacent tensor (garbage downstream).+ // No-op / byte-identical when ne01 % 128 == 0 (no padding).+ if (gid * 2 + 0 < ne01) dst[gid * 2 + 0] = total_sum.s0;+ if (gid * 2 + 1 < ne01) dst[gid * 2 + 1] = total_sum.s1;}}diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.clindex f5c6fb3e84..09bae2d555 100644--- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl@@ -190,6 +190,10 @@ __kernel void kernel_gemv_noshuffle_q8_0_f32(// 1 outputs per fiber in wave 0if (groupId == 0) {dst = (global float*)((global char*)dst + offsetd);- dst[gid] = totalSum;+ // Guard the output row. The x-grid is padded to CEIL_DIV(M,wavesize)*wavesize,+ // so when ne01 is not a multiple of the wave size the tail work-items run past+ // row ne01 and would overrun dst into the adjacent tensor. No-op / byte-identical+ // when ne01 is wave-aligned (no padding).+ if (gid < M) dst[gid] = totalSum;}}