Skip to content

LLM inference in C/C++

C++ ggml

opencl: avoid the vec path in GEMV for unaligned row stride (#25671)

Hongqiang Wanga4ce2595c55e
diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl b/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl
index 9393b54941..b4b03eb11a 100644
--- a/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl
+++ b/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl
@@ -64,7 +64,14 @@ kernel void kernel_mul_mat_f16_f16(
global half * x = (global half *) (src0 + offset_src0);
- if (ne00 < 128) {
+ // The vector path below casts the row pointers to half4, which must be 8-byte aligned.
+ // A row address is r0*nb01 + ..., and a permuted or strided src leaves nb01/nb11
+ // unconstrained -- an odd ne00, say, gives a row that is only 2-byte aligned. Every
+ // src1 row this work-item walks is src1_base + r1*nb11, so require both.
+ const ulong src1_base = (ulong) (src1 + (i12)*nb12 + (i13)*nb13);
+ const bool row_aligned = (((ulong) x) & 7) == 0 && (src1_base & 7) == 0 && (nb11 & 7) == 0;
+
+ if (ne00 < 128 || !row_aligned) {
for (int row = 0; row < N_F16_F16; ++row) {
int r1 = rb + row;
if (r1 >= ne11) {
diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl
index e52d3c6d47..8f3ed9c7b9 100644
--- a/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl
+++ b/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl
@@ -64,7 +64,14 @@ kernel void kernel_mul_mat_f16_f32(
global half * x = (global half *) (src0 + offset_src0);
- if (ne00 < 128) {
+ // The vector path below casts the row pointers to half4/float4, which must be 8- and
+ // 16-byte aligned. A row address is r0*nb01 + ..., and a permuted or strided src leaves
+ // nb01/nb11 unconstrained -- an odd ne00, say, gives a row that is only 2-byte aligned.
+ // Every src1 row this work-item walks is src1_base + r1*nb11, so require both.
+ const ulong src1_base = (ulong) (src1 + (i12)*nb12 + (i13)*nb13);
+ const bool row_aligned = (((ulong) x) & 7) == 0 && (src1_base & 15) == 0 && (nb11 & 15) == 0;
+
+ if (ne00 < 128 || !row_aligned) {
for (int row = 0; row < N_F16_F32; ++row) {
int r1 = rb + row;
if (r1 >= ne11) {
diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl b/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl
index 28d30212cd..eca45615ef 100644
--- a/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl
+++ b/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl
@@ -64,8 +64,15 @@ kernel void kernel_mul_mat_f16_f32_1row(
global half * x = (global half *) (src0 + offset_src0);
global float * y = (global float *) (src1 + offset_src1);
+ // The vector path below casts the row pointers to half4/float4, which must be 8- and
+ // 16-byte aligned. A row address is r0*nb01 + ..., and a permuted or strided src leaves
+ // nb01/nb11 unconstrained -- an odd ne00, say, gives a row that is only 2-byte aligned.
+ // Take the vector path only when the rows this work-item touches are actually aligned;
+ // the scalar loop has no such requirement.
+ const bool row_aligned = (((ulong) x) & 7) == 0 && (((ulong) y) & 15) == 0;
+
float sumf = 0;
- if (ne00 < 128) {
+ if (ne00 < 128 || !row_aligned) {
for (int i = get_sub_group_local_id(); i < ne00; i += get_max_sub_group_size()) {
sumf += (float) x[i] * (float) y[i];
}