LLM inference in C/C++
C++ ggml
opencl: fix q8_0 dp4a mul_mat_id and mark PREC_F32 mul_mat_id unsupported for adreno (#30365)
lhez21a19cd3be83
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cppindex 2abc0bc583..ea7b16cb06 100644--- a/ggml/src/ggml-opencl/ggml-opencl.cpp+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp@@ -9318,6 +9318,12 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te}return false;case GGML_OP_MUL_MAT_ID:+#ifdef GGML_OPENCL_USE_ADRENO_KERNELS+ if (ggml_get_op_params_i32(op, 3) == GGML_PREC_F32 &&+ use_adreno_moe_kernels(backend_ctx, op->src[0])) {+ return false;+ }+#endifif (op->src[0]->type == GGML_TYPE_Q4_0 ||op->src[0]->type == GGML_TYPE_Q8_0 ||op->src[0]->type == GGML_TYPE_MXFP4) {diff --git a/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.clindex d0b191e183..0fdd8d2033 100644--- a/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl+++ b/ggml/src/ggml-opencl/kernels/gemm_moe_q8_1_dp4a.cl@@ -100,6 +100,13 @@ inline int dp4a4(uint w0,uint w1,uint w2,uint w3,uint a0,uint a1,uint a2,uint a3int r=0; r=dot_acc_sat_4x8packed_ss_int(w0,a0,r); r=dot_acc_sat_4x8packed_ss_int(w1,a1,r);r=dot_acc_sat_4x8packed_ss_int(w2,a2,r); r=dot_acc_sat_4x8packed_ss_int(w3,a3,r); return r; }+// Q8_0 and MXFP4 have no min term. Do not multiply zero by an overflowing half activation sum.+#if MOE_QT == 80 || MOE_QT == 82+#define MOE_MIN_TERM(t) 0.0f+#else+#define MOE_MIN_TERM(t) (mn*(float)sh_s[t])+#endif+// One token's two-half dp4a + uniform scale/min epilogue into acc[t].#define MOE_DP4A_T(t) do { \uint4 a0 = vload4(0, &sh_qa[t][0]); \@@ -107,7 +114,7 @@ inline int dp4a4(uint w0,uint w1,uint w2,uint w3,uint a0,uint a1,uint a2,uint a3const int raw1 = dp4a4(qw[0],qw[1],qw[2],qw[3], a0.s0,a0.s1,a0.s2,a0.s3); \const int raw2 = dp4a4(qw[4],qw[5],qw[6],qw[7], a1.s0,a1.s1,a1.s2,a1.s3); \const float a_d = (float)sh_d[t]; \- acc[t] += sc0*a_d*(float)raw1 + sc1*a_d*(float)raw2 - mn*(float)sh_s[t]; \+ acc[t] += sc0*a_d*(float)raw1 + sc1*a_d*(float)raw2 - MOE_MIN_TERM(t); \} while (0)__attribute__((qcom_wave_pair_mode(1)))