Skip to content

LLM inference in C/C++

C++ ggml

opencl: handle OOB write in noshuffle GEMV kernels (odd ne01) (#25640)

Hongqiang Wang00fa7cb284cb
diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl
index 9386bf25a6..1f832cb253 100644
--- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl
+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_nl_f32.cl
@@ -296,7 +296,12 @@ kernel void kernel_gemv_noshuffle_iq4_nl_f32(
// 2 outputs per fiber in wave 0
if (groupId == 0) {
dst = (global float*)((global char*)dst + offsetd);
- vstore2(totalSum, 0, &(dst[gid * 2]));
+ // Guard the two output rows. The x-grid is padded to CEIL_DIV(ne01/2,64)*64,
+ // so when ne01 is not a multiple of 128 the tail row-pairs run past row ne01
+ // and would overrun dst into the adjacent tensor. No-op / byte-identical when
+ // ne01 % 128 == 0 (M/2 already a multiple of 64 -> no padding).
+ if (gid * 2 + 0 < M) dst[gid * 2 + 0] = totalSum.s0;
+ if (gid * 2 + 1 < M) dst[gid * 2 + 1] = totalSum.s1;
}
}
diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl
index e83c5d0689..9efede2941 100644
--- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl
+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl
@@ -116,6 +116,10 @@ __kernel void kernel_gemv_noshuffle_q1_0_f32(
if (groupId == 0) {
dst = (global float*)((global char*)dst + offsetd);
- dst[gid] = totalSum;
+ // Guard the output row. The x-grid is padded to CEIL_DIV(M,wavesize)*wavesize,
+ // so when ne01 is not a multiple of the wave size the tail work-items run past
+ // row ne01 and would overrun dst into the adjacent tensor. No-op / byte-identical
+ // when ne01 is wave-aligned (no padding).
+ if (gid < M) dst[gid] = totalSum;
}
}
diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl
index 1068320691..8de0de1cc3 100644
--- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl
+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32.cl
@@ -268,7 +268,12 @@ __kernel void kernel_gemv_noshuffle_q4_0_f32(
// 2 outputs per fiber in wave 0
if (groupId == 0) {
dst = (global float*)((global char*)dst + offsetd);
- vstore2(totalSum, 0, &(dst[gid * 2]));
+ // Guard the two output rows. The x-grid is padded to CEIL_DIV(ne01/2,64)*64,
+ // so when ne01 is not a multiple of 128 the tail row-pairs run past row ne01
+ // and would overrun dst into the adjacent tensor. No-op / byte-identical when
+ // ne01 % 128 == 0 (M/2 already a multiple of 64 -> no padding).
+ if (gid * 2 + 0 < M) dst[gid * 2 + 0] = totalSum.s0;
+ if (gid * 2 + 1 < M) dst[gid * 2 + 1] = totalSum.s1;
}
}
diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl
index 571a375da7..0dca20f71f 100644
--- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl
+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_0_f32_spec.cl
@@ -262,7 +262,11 @@ __kernel void kernel_gemv_noshuffle_q4_0_f32(
// 2 outputs per fiber in wave 0
if (groupId == 0) {
dst = (global float*)((global char*)dst + offsetd);
- vstore2(totalSum, 0, &(dst[gid * 2]));
+ // Guard the two output rows against the padded x-grid tail overrunning dst.
+ // The current shape specializations are all ne01 % 128 == 0 (no padding), so
+ // this is a no-op / byte-identical today; keep it in lockstep with the base kernel.
+ if (gid * 2 + 0 < ne01) dst[gid * 2 + 0] = totalSum.s0;
+ if (gid * 2 + 1 < ne01) dst[gid * 2 + 1] = totalSum.s1;
}
}
diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl
index fdc1472454..5fa3127806 100644
--- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl
+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_1_f32.cl
@@ -277,7 +277,12 @@ kernel void kernel_gemv_noshuffle_q4_1_f32(
// 2 outputs per fiber in wave 0
if (groupId == 0) {
dst = (global float*)((global char*)dst + offsetd);
- vstore2(totalSum, 0, &(dst[gid * 2]));
+ // Guard the two output rows. The x-grid is padded to CEIL_DIV(ne01/2,64)*64,
+ // so when ne01 is not a multiple of 128 the tail row-pairs run past row ne01
+ // and would overrun dst into the adjacent tensor. No-op / byte-identical when
+ // ne01 % 128 == 0 (M/2 already a multiple of 64 -> no padding).
+ if (gid * 2 + 0 < M) dst[gid * 2 + 0] = totalSum.s0;
+ if (gid * 2 + 1 < M) dst[gid * 2 + 1] = totalSum.s1;
}
}
diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl
index dd1e2b55c0..2eb20e2f76 100644
--- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl
+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q4_k_f32.cl
@@ -312,7 +312,12 @@ kernel void kernel_gemv_noshuffle_q4_k_f32(
// 2 outputs per fiber in wave 0
if (groupId == 0) {
dst = (global float*)((global char*)dst + offsetd);
- vstore2(totalSum, 0, &(dst[gid * 2]));
+ // Guard the two output rows. The x-grid is padded to CEIL_DIV(ne01/2,64)*64,
+ // so when ne01 is not a multiple of 128 the tail row-pairs run past row ne01
+ // and would overrun dst into the adjacent tensor. No-op / byte-identical when
+ // ne01 % 128 == 0 (M/2 already a multiple of 64 -> no padding).
+ if (gid * 2 + 0 < M) dst[gid * 2 + 0] = totalSum.s0;
+ if (gid * 2 + 1 < M) dst[gid * 2 + 1] = totalSum.s1;
}
}
diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl
index c228f717a9..7dbf5a3bbb 100644
--- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl
+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl
@@ -285,7 +285,12 @@ __kernel void kernel_gemv_noshuffle_q5_0_f32(
// 2 outputs per fiber in wave 0
if (groupId == 0) {
dst = (global float*)((global char*)dst + offsetd);
- vstore2(totalSum, 0, &(dst[gid * 2]));
+ // Guard the two output rows. The x-grid is padded to CEIL_DIV(ne01/2,64)*64,
+ // so when ne01 is not a multiple of 128 the tail row-pairs run past row ne01
+ // and would overrun dst into the adjacent tensor. No-op / byte-identical when
+ // ne01 % 128 == 0 (M/2 already a multiple of 64 -> no padding).
+ if (gid * 2 + 0 < M) dst[gid * 2 + 0] = totalSum.s0;
+ if (gid * 2 + 1 < M) dst[gid * 2 + 1] = totalSum.s1;
}
}
diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl
index daf1308ea4..ba0e2a7115 100644
--- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl
+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl
@@ -288,7 +288,12 @@ __kernel void kernel_gemv_noshuffle_q5_1_f32(
// 2 outputs per fiber in wave 0
if (groupId == 0) {
dst = (global float*)((global char*)dst + offsetd);
- vstore2(totalSum, 0, &(dst[gid * 2]));
+ // Guard the two output rows. The x-grid is padded to CEIL_DIV(ne01/2,64)*64,
+ // so when ne01 is not a multiple of 128 the tail row-pairs run past row ne01
+ // and would overrun dst into the adjacent tensor. No-op / byte-identical when
+ // ne01 % 128 == 0 (M/2 already a multiple of 64 -> no padding).
+ if (gid * 2 + 0 < M) dst[gid * 2 + 0] = totalSum.s0;
+ if (gid * 2 + 1 < M) dst[gid * 2 + 1] = totalSum.s1;
}
}
diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl
index c40db16663..446f465338 100644
--- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl
+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_k_f32.cl
@@ -321,6 +321,11 @@ kernel void kernel_gemv_noshuffle_q5_k_f32(
// 2 outputs per fiber in wave 0
if (groupId == 0) {
dst = (global float*)((global char*)dst + offsetd);
- vstore2(totalSum, 0, &(dst[gid * 2]));
+ // Guard the two output rows. The x-grid is padded to CEIL_DIV(ne01/2,64)*64,
+ // so when ne01 is not a multiple of 128 the tail row-pairs run past row ne01
+ // and would overrun dst into the adjacent tensor. No-op / byte-identical when
+ // ne01 % 128 == 0 (M/2 already a multiple of 64 -> no padding).
+ if (gid * 2 + 0 < M) dst[gid * 2 + 0] = totalSum.s0;
+ if (gid * 2 + 1 < M) dst[gid * 2 + 1] = totalSum.s1;
}
}
diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl
index 6f89cf968b..51682ecebb 100644
--- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl
+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q6_k_f32.cl
@@ -288,6 +288,11 @@ kernel void kernel_gemv_noshuffle_q6_K_f32(
if (grp == 0) {
dst = (global float*)((global char*)dst + offsetd);
- vstore2(total_sum, 0, &(dst[gid * 2]));
+ // Guard the two output rows. The x-grid is padded to CEIL_DIV(ne01/2,64)*64,
+ // so when ne01 is not a multiple of 128 the tail row-pairs run past row ne01
+ // and would overrun dst into the adjacent tensor (garbage downstream).
+ // No-op / byte-identical when ne01 % 128 == 0 (no padding).
+ if (gid * 2 + 0 < ne01) dst[gid * 2 + 0] = total_sum.s0;
+ if (gid * 2 + 1 < ne01) dst[gid * 2 + 1] = total_sum.s1;
}
}
diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl
index f5c6fb3e84..09bae2d555 100644
--- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl
+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q8_0_f32.cl
@@ -190,6 +190,10 @@ __kernel void kernel_gemv_noshuffle_q8_0_f32(
// 1 outputs per fiber in wave 0
if (groupId == 0) {
dst = (global float*)((global char*)dst + offsetd);
- dst[gid] = totalSum;
+ // Guard the output row. The x-grid is padded to CEIL_DIV(M,wavesize)*wavesize,
+ // so when ne01 is not a multiple of the wave size the tail work-items run past
+ // row ne01 and would overrun dst into the adjacent tensor. No-op / byte-identical
+ // when ne01 is wave-aligned (no padding).
+ if (gid < M) dst[gid] = totalSum;
}
}