LLM inference in C/C++
C++ ggml
opencl: avoid the vec path in GEMV for unaligned row stride (#25671)
Hongqiang Wanga4ce2595c55e
diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl b/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.clindex 9393b54941..b4b03eb11a 100644--- a/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl+++ b/ggml/src/ggml-opencl/kernels/mul_mv_f16_f16.cl@@ -64,7 +64,14 @@ kernel void kernel_mul_mat_f16_f16(global half * x = (global half *) (src0 + offset_src0);- if (ne00 < 128) {+ // The vector path below casts the row pointers to half4, which must be 8-byte aligned.+ // A row address is r0*nb01 + ..., and a permuted or strided src leaves nb01/nb11+ // unconstrained -- an odd ne00, say, gives a row that is only 2-byte aligned. Every+ // src1 row this work-item walks is src1_base + r1*nb11, so require both.+ const ulong src1_base = (ulong) (src1 + (i12)*nb12 + (i13)*nb13);+ const bool row_aligned = (((ulong) x) & 7) == 0 && (src1_base & 7) == 0 && (nb11 & 7) == 0;++ if (ne00 < 128 || !row_aligned) {for (int row = 0; row < N_F16_F16; ++row) {int r1 = rb + row;if (r1 >= ne11) {diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.clindex e52d3c6d47..8f3ed9c7b9 100644--- a/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl+++ b/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32.cl@@ -64,7 +64,14 @@ kernel void kernel_mul_mat_f16_f32(global half * x = (global half *) (src0 + offset_src0);- if (ne00 < 128) {+ // The vector path below casts the row pointers to half4/float4, which must be 8- and+ // 16-byte aligned. A row address is r0*nb01 + ..., and a permuted or strided src leaves+ // nb01/nb11 unconstrained -- an odd ne00, say, gives a row that is only 2-byte aligned.+ // Every src1 row this work-item walks is src1_base + r1*nb11, so require both.+ const ulong src1_base = (ulong) (src1 + (i12)*nb12 + (i13)*nb13);+ const bool row_aligned = (((ulong) x) & 7) == 0 && (src1_base & 15) == 0 && (nb11 & 15) == 0;++ if (ne00 < 128 || !row_aligned) {for (int row = 0; row < N_F16_F32; ++row) {int r1 = rb + row;if (r1 >= ne11) {diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl b/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.clindex 28d30212cd..eca45615ef 100644--- a/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl+++ b/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_1row.cl@@ -64,8 +64,15 @@ kernel void kernel_mul_mat_f16_f32_1row(global half * x = (global half *) (src0 + offset_src0);global float * y = (global float *) (src1 + offset_src1);+ // The vector path below casts the row pointers to half4/float4, which must be 8- and+ // 16-byte aligned. A row address is r0*nb01 + ..., and a permuted or strided src leaves+ // nb01/nb11 unconstrained -- an odd ne00, say, gives a row that is only 2-byte aligned.+ // Take the vector path only when the rows this work-item touches are actually aligned;+ // the scalar loop has no such requirement.+ const bool row_aligned = (((ulong) x) & 7) == 0 && (((ulong) y) & 15) == 0;+float sumf = 0;- if (ne00 < 128) {+ if (ne00 < 128 || !row_aligned) {for (int i = get_sub_group_local_id(); i < ne00; i += get_max_sub_group_size()) {sumf += (float) x[i] * (float) y[i];}