LLM inference in C/C++
C++ ggml
opencl: Q6_K GEMM/GEMV fix for ne01 of weights that are not multiples of 128. (#25464)
Hongqiang Wang92366df30d4e
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cppindex 10e183530e..38600f31a7 100644--- a/ggml/src/ggml-opencl/ggml-opencl.cpp+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp@@ -6242,8 +6242,14 @@ inline bool use_adreno_kernels(const ggml_backend_opencl_context *backend_ctx, cthreshold_ne0 = 128;threshold_ne1 = 128;}- return tensor->ne[0] >= threshold_ne0 && tensor->ne[1] >= threshold_ne1 &&+ bool threashold_ok = tensor->ne[0] >= threshold_ne0 && tensor->ne[1] >= threshold_ne1 &&tensor->ne[2] == 1 && tensor->ne[3] == 1;++ // q6_K adreno kernels requires ne1 is multiple of 128+ if (tensor->type == GGML_TYPE_Q6_K) {+ return threashold_ok && tensor->ne[1] % 128 == 0;+ }+ return threashold_ok;}inline bool use_adreno_moe_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) {@@ -6273,6 +6279,19 @@ static inline bool use_flat_gemv_for_large_m_q6_K(const ggml_tensor *tensor) {// threshold is well above typical hidden/FFN dims, but below typical vocab sizes.// q6_K flat gemv is worse for smaller K; 2048 seems to be a reasonable threshold.// note that this forces large M weights to use LM GEMM.+ // The noshuffle (transposed-weight) layout packs 2 rows per 32-bit texel and the+ // gemv reads it with a ne01/2 texel stride and an exact-cover dispatch of+ // ceil(ne01/2 / 64)*64 work-items with no store guard; the gemm uses 4-row tiles.+ // It is therefore only correct for ne01 % 128 == 0: an odd ne01 (e.g. granitemoe+ // lm_head [1536, 49155] -- odd vocab) truncates the texel stride, misaligning every+ // odd column of the transposed layout (gross garbage) and dropping the last row;+ // other non-multiples over-dispatch and write past the end of dst. Route such+ // tensors to the flat GEMV + regular convert; the matching GEMM (ne1>1) falls back+ // to CPU (see supports_op). All standard even-vocab/hidden dims are multiples of+ // 128 and keep the noshuffle path.+ if ((tensor->ne[1] % 128 != 0) && tensor->ne[2] == 1 && tensor->ne[3] == 1) {+ return true;+ }return tensor->ne[1] >= 32768 && tensor->ne[0] >= 2048 && tensor->ne[2] == 1 && tensor->ne[3] == 1;}@@ -9776,12 +9795,30 @@ static bool ggml_backend_opencl_buffer_type_supports_backend(ggml_backend_bufferUNUSED(buft);}+static size_t ggml_backend_opencl_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {+ size_t size = ggml_nbytes(tensor);+#ifdef GGML_OPENCL_SOA_Q+ // set_tensor carves quantized weights into per-component subbuffers (d/q,+ // ql/qh/s/d, ...) whose origins are each rounded up to the device base+ // alignment. When a component's size is not a multiple of the alignment+ // (e.g. q6_K [1536,49155]: size_s = 49155*96 leaves a 96-byte gap at 128-byte+ // alignment), the aligned carve extends past ggml_nbytes and the last+ // subbuffer would overlap the next tensor in the pool. Reserve the worst-case+ // carve slack: at most 5 components (q5_K), i.e. 4 aligned gaps.+ if (ggml_is_quantized(tensor->type)) {+ ggml_backend_opencl_device_context * dev_ctx = (ggml_backend_opencl_device_context *) buft->device->context;+ size += 4 * dev_ctx->backend_ctx->alignment;+ }+#endif // GGML_OPENCL_SOA_Q+ return size;+}+static ggml_backend_buffer_type_i ggml_backend_opencl_buffer_type_interface = {/* .get_name = */ ggml_backend_opencl_buffer_type_get_name,/* .alloc_buffer = */ ggml_backend_opencl_buffer_type_alloc_buffer,/* .get_alignment = */ ggml_backend_opencl_buffer_type_get_alignment,/* .get_max_size = */ ggml_backend_opencl_buffer_type_get_max_size,- /* .get_alloc_size = */ NULL,+ /* .get_alloc_size = */ ggml_backend_opencl_buffer_type_get_alloc_size,/* .is_host = */ NULL,};