LLM inference in C/C++
C++ ggml
opencl: fix a dp4a bug for devices where cl_khr_integer_dot_product is unavailable (#25639)
Hongqiang Wang00e79f6fb146
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cppindex f283f65690..7b69175921 100644--- a/ggml/src/ggml-opencl/ggml-opencl.cpp+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp@@ -532,6 +532,7 @@ struct ggml_backend_opencl_context {bool fp16_support;bool has_vector_subgroup_broadcast;bool has_subgroup_shuffle = false; // cl_khr_subgroup_shuffle or cl_qcom_subgroup_shuffle+ bool has_integer_dot = false; // cl_khr_integer_dot_product or cl_qcom_dot_product8bool has_qcom_subgroup_shuffle = false; // specifically cl_qcom_subgroup_shufflebool disable_fusion;@@ -834,7 +835,7 @@ struct ggml_backend_opencl_context {cl_kernel kernel_gemv_moe_q5_1_f32_ns, kernel_gemm_moe_q5_1_f32_ns;cl_kernel kernel_gemv_moe_q4_k_f32_ns, kernel_gemm_moe_q4_k_f32_ns, kernel_gemm_moe_q4_k_f32_ns_bin;cl_kernel kernel_gemv_moe_q4_k_f32_ns_wimg = nullptr; // weight-as-texture MoE decode GEMV (opt-in)- cl_kernel kernel_gemm_moe_q4_k_q8_1_dp4a; // dp4a (int8) prefill GEMM variant+ cl_kernel kernel_gemm_moe_q4_k_q8_1_dp4a = nullptr; // dp4a (int8) prefill GEMM variantcl_kernel kernel_moe_reorder_quant_a_q8_1; // fused reorder + q8_1 quant for the dp4a GEMMcl_kernel kernel_gemm_moe_q8_1_dp4a_q80 = nullptr; // generic dp4a MoE GEMM (MOE_QT=80), opt-incl_kernel kernel_moe_expand_scale_q8_0 = nullptr; // q8_0 per-block d -> uniform scale[16]@@ -844,12 +845,12 @@ struct ggml_backend_opencl_context {cl_kernel kernel_moe_expand_scale_q5_K = nullptr; // q5_K 6-bit s[] -> uniform scale[16]/min[8]cl_kernel kernel_gemv_moe_q5_k_f32_ns, kernel_gemm_moe_q5_k_f32_ns;cl_kernel kernel_gemv_moe_q6_k_f32_ns, kernel_gemm_moe_q6_k_f32_ns;- cl_kernel kernel_gemm_moe_q6_k_q8_1_dp4a; // dp4a (int8) q6_K MoE prefill GEMM+ cl_kernel kernel_gemm_moe_q6_k_q8_1_dp4a = nullptr; // dp4a (int8) q6_K MoE prefill GEMMcl_kernel kernel_gemv_moe_mxfp4_f32, kernel_gemm_moe_mxfp4_f32;cl_kernel kernel_gemv_moe_mxfp4_f32_ns, kernel_gemm_moe_mxfp4_f32_ns, kernel_gemm_moe_mxfp4_f32_ns_bin;cl_kernel kernel_gemv_moe_mxfp4_f32_ns_wimg = nullptr; // weight-as-texture MoE decode GEMV- cl_kernel kernel_gemm_moe_mxfp4_q8_1_dp4a; // dp4a (int8) mxfp4 MoE prefill GEMM- cl_kernel kernel_gemm_moe_q4_0_q8_1_dp4a; // dp4a (int8) q4_0 MoE prefill GEMM+ cl_kernel kernel_gemm_moe_mxfp4_q8_1_dp4a = nullptr; // dp4a (int8) mxfp4 MoE prefill GEMM+ cl_kernel kernel_gemm_moe_q4_0_q8_1_dp4a = nullptr; // dp4a (int8) q4_0 MoE prefill GEMMcl_kernel kernel_moe_reorder_b;cl_kernel kernel_moe_histogram, kernel_moe_scan, kernel_moe_fill, kernel_moe_scatter;cl_kernel kernel_moe_combine_f32 = nullptr; // fused router-weight mul + cross-expert sum@@ -1037,10 +1038,10 @@ struct ggml_backend_opencl_context {cl_kernel kernel_gemv_noshuffle_q1_0_f32;cl_kernel kernel_gemv_noshuffle_q4_k_f32;cl_kernel kernel_gemm_noshuffle_q4_k_f32;- cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a; // dp4a (int8) dense prefill GEMM- cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg; // dp4a dense prefill GEMM, weights via texture (X1 opt-in)- cl_kernel kernel_gemm_noshuffle_q5_k_q8_1_dp4a; // dp4a (int8) dense q5_K prefill GEMM- cl_kernel kernel_gemm_noshuffle_q6_k_q8_1_dp4a; // dp4a (int8) dense q6_K prefill GEMM+ cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a = nullptr; // dp4a (int8) dense prefill GEMM+ cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg = nullptr; // dp4a dense prefill GEMM, weights via texture (X1 opt-in)+ cl_kernel kernel_gemm_noshuffle_q5_k_q8_1_dp4a = nullptr; // dp4a (int8) dense q5_K prefill GEMM+ cl_kernel kernel_gemm_noshuffle_q6_k_q8_1_dp4a = nullptr; // dp4a (int8) dense q6_K prefill GEMMcl_kernel kernel_quant_a_q8_1; // plain activation q8_1 pre-passcl_kernel kernel_gemv_noshuffle_q6_K_f32;cl_kernel kernel_gemm_noshuffle_q6_K_f32;@@ -3490,7 +3491,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {}// gemm_noshuffle_q5_0_q8_1_dp4a (dp4a dense q5_0 prefill GEMM)- {+ if (backend_ctx->has_integer_dot) {#ifdef GGML_OPENCL_EMBED_KERNELSconst std::string kernel_src {#include "gemm_noshuffle_q5_0_q8_1_dp4a.cl.h"@@ -3580,7 +3581,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {}// gemm_noshuffle_iq4_nl_q8_1_dp4a (dp4a dense IQ4_NL prefill GEMM)- {+ if (backend_ctx->has_integer_dot) {#ifdef GGML_OPENCL_EMBED_KERNELSconst std::string kernel_src {#include "gemm_noshuffle_iq4_nl_q8_1_dp4a.cl.h"@@ -3595,7 +3596,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {}// gemm_noshuffle_q4_0_q8_1_dp4a (dp4a dense q4_0 prefill GEMM)- {+ if (backend_ctx->has_integer_dot) {#ifdef GGML_OPENCL_EMBED_KERNELSconst std::string kernel_src {#include "gemm_noshuffle_q4_0_q8_1_dp4a.cl.h"@@ -3708,7 +3709,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {}// gemm_noshuffle_q4_k_q8_1_dp4a (dp4a dense prefill GEMM)- {+ if (backend_ctx->has_integer_dot) {#ifdef GGML_OPENCL_EMBED_KERNELSconst std::string kernel_src {#include "gemm_noshuffle_q4_k_q8_1_dp4a.cl.h"@@ -3730,7 +3731,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {}// gemm_noshuffle_q8_0_q8_1_dp4a (dp4a dense q8_0 prefill GEMM)- {+ if (backend_ctx->has_integer_dot) {#ifdef GGML_OPENCL_EMBED_KERNELSconst std::string kernel_src {#include "gemm_noshuffle_q8_0_q8_1_dp4a.cl.h"@@ -3746,7 +3747,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {}// gemm_noshuffle_q5_k_q8_1_dp4a (dp4a dense prefill GEMM for q5_K)- {+ if (backend_ctx->has_integer_dot) {#ifdef GGML_OPENCL_EMBED_KERNELSconst std::string kernel_src {#include "gemm_noshuffle_q5_k_q8_1_dp4a.cl.h"@@ -3761,7 +3762,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {}// gemm_noshuffle_q6_k_q8_1_dp4a (dp4a dense prefill GEMM for q6_K ffn_down/output)- {+ if (backend_ctx->has_integer_dot) {#ifdef GGML_OPENCL_EMBED_KERNELSconst std::string kernel_src {#include "gemm_noshuffle_q6_k_q8_1_dp4a.cl.h"@@ -4091,7 +4092,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {}// gemm_moe_q4_k_q8_1_dp4a (dp4a prefill GEMM)- {+ if (backend_ctx->has_integer_dot) {#ifdef GGML_OPENCL_EMBED_KERNELSconst std::string kernel_src {#include "gemm_moe_q4_k_q8_1_dp4a.cl.h"@@ -4108,7 +4109,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {}// gemm_moe_mxfp4_q8_1_dp4a (dp4a prefill GEMM)- {+ if (backend_ctx->has_integer_dot) {#ifdef GGML_OPENCL_EMBED_KERNELSconst std::string kernel_src {#include "gemm_moe_mxfp4_q8_1_dp4a.cl.h"@@ -4125,7 +4126,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {}// gemm_moe_q4_0_q8_1_dp4a (dp4a prefill GEMM)- {+ if (backend_ctx->has_integer_dot) {#ifdef GGML_OPENCL_EMBED_KERNELSconst std::string kernel_src {#include "gemm_moe_q4_0_q8_1_dp4a.cl.h"@@ -4142,7 +4143,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {}// gemm_moe_q8_1_dp4a (generic dp4a MoE GEMM; MOE_QT=80 -> q8_0 expert variant)- {+ if (backend_ctx->has_integer_dot) {#ifdef GGML_OPENCL_EMBED_KERNELSconst std::string kernel_src {#include "gemm_moe_q8_1_dp4a.cl.h"@@ -4256,7 +4257,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {}// gemm_moe_q6_k_q8_1_dp4a (dp4a q6_K MoE prefill GEMM)- {+ if (backend_ctx->has_integer_dot) {#ifdef GGML_OPENCL_EMBED_KERNELSconst std::string kernel_src {#include "gemm_moe_q6_k_q8_1_dp4a.cl.h"@@ -5602,6 +5603,8 @@ static void ggml_opencl_print_backend_info(ggml_backend_opencl_device_context *backend_ctx->has_subgroup_shuffle ? "true" : "false");GGML_LOG_INFO("ggml_opencl: device FP16 support: %s\n",backend_ctx->fp16_support ? "true" : "false");+ GGML_LOG_INFO("ggml_opencl: khr dot product support: %s\n",+ backend_ctx->has_integer_dot ? "true" : "false");GGML_LOG_INFO("ggml_opencl: mem base addr align: %u\n",backend_ctx->alignment);GGML_LOG_INFO("ggml_opencl: global mem size: %zu MB\n",@@ -5810,6 +5813,12 @@ static ggml_backend_opencl_context * ggml_cl_init(ggml_backend_dev_t dev) {strstr(ext_buffer, "cl_khr_subgroup_shuffle") != NULL ||backend_ctx->has_qcom_subgroup_shuffle;+ // check for cl_khr_integer_dot_product+ // cl_qcom_dot_product8 uses signed * unsigned+ // while cl_khr_integer_dot_product uses signed * signed -- we stick with khr for now+ backend_ctx->has_integer_dot =+ strstr(ext_buffer, "cl_khr_integer_dot_product") != NULL;+cl_uint base_align_in_bits;CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_MEM_BASE_ADDR_ALIGN, sizeof(cl_uint), &base_align_in_bits, NULL));GGML_ASSERT(base_align_in_bits % 8u == 0);@@ -15819,18 +15828,14 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_tCL_CHECK(clReleaseMemObject(b_sub_buf));CL_CHECK(clReleaseMemObject(b_img));} else {- // dp4a (int8) dense prefill GEMM: quant activations to q8_1, then the int8- // dp4a inner-loop GEMM, in place of the transpose + f16 half-dot kernel.- // q4_0 = d*(q-8); mirrors the IQ4_NL/q8_0 dense dp4a paths (+ the sum term).- // OPT-IN / DEFAULT OFF: correct, but neutral on X2E. q4_0's dequant- // ((q-8)*scale) is already trivial so the f16 GEMM is weight-BW-bound and the- // int8 ALU win has nothing to beat -- same as q5_0 dense (unlike IQ4_NL, whose- // codebook dequant is expensive enough for dp4a to help). Kept for A/B; force- // on with GGML_OPENCL_Q4_0_DENSE_DP4A=1. Needs N>8, K%32==0, M%64==0.+ // dp4a (int8) dense prefill GEMM, default offstatic const char * q4_0_dense_dp4a_env = getenv("GGML_OPENCL_Q4_0_DENSE_DP4A");- const bool q4_0_dense_dp4a_on = q4_0_dense_dp4a_env+ bool q4_0_dense_dp4a_on = q4_0_dense_dp4a_env? (atoi(q4_0_dense_dp4a_env) != 0): false;+ // dot prod has to be available+ q4_0_dense_dp4a_on = backend_ctx->has_integer_dot && q4_0_dense_dp4a_on;+if (q4_0_dense_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a&& N > 8 && (K % 32 == 0) && (M % 64 == 0)) {cl_mem a_sub = nullptr;@@ -16253,27 +16258,16 @@ static void ggml_cl_mul_mat_q5_0_f32_adreno(ggml_backend_t backend, const ggml_tCL_CHECK(clReleaseMemObject(b_sub_buf));CL_CHECK(clReleaseMemObject(b_img));} else {- // dp4a (int8) dense q5_0 prefill GEMM. Quantizes the [N,K] activations to- // q8_1 and runs the int8 dot instead of the f16 half-dot. Large-batch- // (ne1>8) only. q5_0 weight = (x-16)*d (x = nibble | hi<<4); x packed as a- // 0..31 byte (dp4a), the -16 centering folded into a single min term- // (d*16) via the q8_1 block sum. Reads the qs/qh/d buffers byte-identically- // to the f16 kernel (greedy byte-identical, MUL_MAT NMSE-OK).- //- // OPT-IN / DEFAULT OFF. Unlike q8_0/q4_K dense, dp4a is not a win for q5_0 on- // X2E: the q5_0 model is bottlenecked elsewhere, so the dense-GEMM int8 win- // has nothing to surface and the q8_1 prepass slightly hurts. Kept correct +- // opt-in for the X1 A/B (different texture-cache dynamic) and the- // weight-texture variant. Env: GGML_OPENCL_Q5_DENSE_DP4A=1.- // Weight-as-texture variant (X1 lever): routes the dominant qs nibble plane- // through an image1d_buffer (qh stays a buffer). Opt-in- // GGML_OPENCL_Q5_DENSE_DP4A_WIMG; when set it also forces the dp4a path on.+ // dp4a (int8) dense q5_0 prefill GEMM, default offstatic const char * q5_dense_dp4a_env = getenv("GGML_OPENCL_Q5_DENSE_DP4A");static const char * q5_dense_wimg_env = getenv("GGML_OPENCL_Q5_DENSE_DP4A_WIMG");const bool q5_dense_wimg_on = q5_dense_wimg_env && (atoi(q5_dense_wimg_env) != 0);- const bool q5_dense_dp4a_on = q5_dense_wimg_on+ bool q5_dense_dp4a_on = q5_dense_wimg_on? true: (q5_dense_dp4a_env && (atoi(q5_dense_dp4a_env) != 0));+ // dot prod has to be available+ q5_dense_dp4a_on = backend_ctx->has_integer_dot && q5_dense_dp4a_on;+if (q5_dense_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q5_0_q8_1_dp4a&& N > 8 && (K % 32 == 0) && (M % 64 == 0)) {cl_mem a_sub = nullptr;@@ -16708,15 +16702,14 @@ static void ggml_cl_mul_mat_iq4_nl_f32_adreno(ggml_backend_t backend, const ggml} else {// dp4a (int8) dense IQ4_NL prefill GEMM. Quantizes the [N,K] activations to// q8_1 and runs the int8 dot instead of the f16 half-dot. Large-batch- // (ne1>8) only. IQ4_NL weight = kvalues[nibble]*d; the codebook value IS the- // int8 (no min term), so this is the q8_0 dense case plus a nibble->int8 LUT- // unpack. Reads the q/d buffers byte-identically to the f16 kernel. No bin- // kernel for IQ4_NL -> baseline is f16, default ON for X2E (like q4_K/q6_K- // dense dp4a). X1 stays on f16. Env: GGML_OPENCL_IQ4NL_DENSE_DP4A.+ // (ne1>8) onlystatic const char * iq4nl_dense_dp4a_env = getenv("GGML_OPENCL_IQ4NL_DENSE_DP4A");- const bool iq4nl_dense_dp4a_on = iq4nl_dense_dp4a_env+ bool iq4nl_dense_dp4a_on = iq4nl_dense_dp4a_env? (atoi(iq4nl_dense_dp4a_env) != 0): (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);+ // dot prod has to be available+ iq4nl_dense_dp4a_on = backend_ctx->has_integer_dot && iq4nl_dense_dp4a_on;+if (iq4nl_dense_dp4a_on && backend_ctx->kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a&& N > 8 && (K % 32 == 0) && (M % 64 == 0)) {cl_mem a_sub = nullptr;@@ -16966,13 +16959,15 @@ static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_tstatic const char * q8_dense_wimg_env = getenv("GGML_OPENCL_Q8_DENSE_DP4A_WIMG");const bool q8_dense_wimg_on = q8_dense_wimg_env && (atoi(q8_dense_wimg_env) != 0);- const bool q8_bin_loaded = (backend_ctx->kernel_gemm_noshuffle_q8_0_f32_bin != nullptr);+ const bool q8_bin_loaded = (backend_ctx->kernel_gemm_noshuffle_q8_0_f32_bin != nullptr);// bin kernel takes precedence- const bool q8_dense_dp4a_on = q8_dense_wimg_on+ bool q8_dense_dp4a_on = q8_dense_wimg_on? true: q8_dense_dp4a_env? (atoi(q8_dense_dp4a_env) != 0): (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && !q8_bin_loaded);+ // dot prod has to be available+ q8_dense_dp4a_on = backend_ctx->has_integer_dot && q8_dense_dp4a_on;if (q8_dense_dp4a_on && backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a&& N > 8 && (K % 32 == 0) && (M % 64 == 0)) {@@ -17379,13 +17374,16 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno(ggml_backend_t backend, const ggml_tstatic const char * q4k_dense_dp4a_env = getenv("GGML_OPENCL_Q4K_DENSE_DP4A");static const char * q4k_dense_wimg_env = getenv("GGML_OPENCL_Q4K_DENSE_DP4A_WIMG");- const bool q4k_dense_wimg_on = q4k_dense_wimg_env && (atoi(q4k_dense_wimg_env) != 0);- const bool q4k_dense_dp4a_on = q4k_dense_wimg_on+ const bool q4k_dense_wimg_on = q4k_dense_wimg_env && (atoi(q4k_dense_wimg_env) != 0);+ bool q4k_dense_dp4a_on = q4k_dense_wimg_on? true: q4k_dense_dp4a_env? (atoi(q4k_dense_dp4a_env) != 0): (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);+ // dp4 has to be available+ q4k_dense_dp4a_on = backend_ctx->has_integer_dot && q4k_dense_dp4a_on;+// Min N for the dp4a prefill GEMM, default 9, i.e., ne1 > 8static const char * q4k_dp4a_minn_env = getenv("GGML_OPENCL_Q4K_DP4A_MINN");const int q4k_dp4a_minn = q4k_dp4a_minn_env ? atoi(q4k_dp4a_minn_env) : 9;@@ -17608,9 +17606,11 @@ static void ggml_cl_mul_mat_q6_K_f32_adreno(ggml_backend_t backend, const ggml_t// dp4a (int8) dense q6_K prefill GEMMstatic const char * q6k_dense_dp4a_env = getenv("GGML_OPENCL_Q6K_DENSE_DP4A");- static const bool q6k_dense_dp4a_on = (q6k_dense_dp4a_env != nullptr)+ bool q6k_dense_dp4a_on = (q6k_dense_dp4a_env != nullptr)? (atoi(q6k_dense_dp4a_env) != 0): (backend_ctx->adreno_gen != ADRENO_GPU_GEN::X1E);+ // dot prod has to be available+ q6k_dense_dp4a_on = backend_ctx->has_integer_dot && q6k_dense_dp4a_on;const bool is_output_w_dp4a = strncmp(src0->name, "output", 6) == 0 ||strncmp(src0->name, "token_embd", 10) == 0;@@ -17901,9 +17901,11 @@ static void ggml_cl_mul_mat_q5_K_f32_adreno(ggml_backend_t backend, const ggml_t// dp4a (int8) dense q5_K prefill GEMMstatic const char * q5k_dense_dp4a_env = getenv("GGML_OPENCL_Q5K_DENSE_DP4A");- const bool q5k_dense_dp4a_on = q5k_dense_dp4a_env+ bool q5k_dense_dp4a_on = q5k_dense_dp4a_env? (atoi(q5k_dense_dp4a_env) != 0): (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);+ // dot prod has to be available+ q5k_dense_dp4a_on = backend_ctx->has_integer_dot && q5k_dense_dp4a_on;if (q5k_dense_dp4a_on && ne1 > 8 && (ne00 % 32 == 0) && (ne01 % 64 == 0)) {const int Mm = ne01, Nn = ne1, Kk = ne00;@@ -20640,6 +20642,8 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,bool use_moe_dp4a = q4_0_moe_dp4a_env? (atoi(q4_0_moe_dp4a_env) != 0): (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);+ // dot prod has to be available+ use_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a;// bin kernel takes precedenceuse_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_q4_0_f32_ns_bin == nullptr;@@ -21815,6 +21819,8 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,bool use_moe_dp4a = (q4k_moe_dp4a_env != nullptr)? (atoi(q4k_moe_dp4a_env) != 0): (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E);+ // dot prod has to be available+ use_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a;// bin kernel takes precedenceuse_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin == nullptr;@@ -22316,10 +22322,12 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,// dp4a (int8) q6_K MoE prefill GEMMstatic const char * q6k_moe_dp4a_env = getenv("GGML_OPENCL_Q6K_MOE_DP4A");- static const bool use_moe_dp4a = (q6k_moe_dp4a_env != nullptr)+ bool use_moe_dp4a = (q6k_moe_dp4a_env != nullptr)? (atoi(q6k_moe_dp4a_env) != 0): (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E|| backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E);+ // dot prod has to be available+ use_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a;cl_buffer_region region;region.origin = 0;@@ -22569,6 +22577,8 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,bool use_moe_dp4a = mxfp4_moe_dp4a_env? (atoi(mxfp4_moe_dp4a_env) != 0): (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E);+ // dot prod has to be available+ use_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a;// bin kernel takes precedenceuse_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_mxfp4_f32_ns_bin == nullptr;