LLM inference in C/C++
C++ ggml
opencl: fix q6_K flat mul_mat for Adreno A6x/A7x GPUs with older E031 compilers (#26476)
Hongqiang Wang6503355df0eb
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cppindex d169f33896..49cd9fd355 100644--- a/ggml/src/ggml-opencl/ggml-opencl.cpp+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp@@ -582,6 +582,8 @@ struct ggml_backend_opencl_context {bool adreno_use_bin_kernels;get_adreno_bin_kernel_func_t get_adreno_bin_kernel_func = nullptr;ggml_cl_compiler_version adreno_cl_compiler_version;+ // The q6_K flat mul_mat codegen workarounds are needed by old E031 compilers only.+ bool q6_k_flat_old_compiler;std::string kernel_compile_opts; // cached for lazy-compiled kernels.@@ -1931,8 +1933,14 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {#elseconst std::string kernel_src = read_file("mul_mv_q6_k_f32_flat.cl");#endif+ // The codegen workarounds in this kernel are a measured 13-20% loss on+ // compilers that do not need them, so only the affected ones build them;+ // everyone else gets the original source.+ const std::string q6k_opts = backend_ctx->q6_k_flat_old_compiler+ ? compile_opts + " -DADRENO_OLD_COMPILER=1"+ : compile_opts;cl_program prog =- build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);+ build_program_from_source(backend_ctx, kernel_src.c_str(), q6k_opts);CL_CHECK((backend_ctx->kernel_mul_mv_q6_K_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q6_K_f32_flat", &err), err));CL_CHECK(clReleaseProgram(prog));@@ -5917,6 +5925,16 @@ static ggml_backend_opencl_context * ggml_cl_init(ggml_backend_dev_t dev) {(backend_ctx->adreno_cl_compiler_version.type == E031 && backend_ctx->adreno_cl_compiler_version.major >= 47) ||(backend_ctx->adreno_cl_compiler_version.type == DX && backend_ctx->adreno_cl_compiler_version.major >= 17);+ // The q6_K flat mul_mat miscompile is a defect of the older E031 compilers, not a+ // property of any GPU generation: it reproduces on E031.38 (Adreno 642L) and E031.41+ // (Adreno 740) and is fixed by E031.45 (Adreno 619). Gate on the compiler so parts+ // that do not need the workarounds do not pay for them. The explicit type check is+ // required: newer_than_or_same() is false for every non-E031 compiler, so negating it+ // alone would enable the workarounds on E17/DX.+ backend_ctx->q6_k_flat_old_compiler =+ backend_ctx->adreno_cl_compiler_version.type == E031 &&+ !backend_ctx->adreno_cl_compiler_version.newer_than_or_same(E031, 45, 0, 0);+size_t ext_str_size;clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, 0, NULL, &ext_str_size);char *ext_buffer = (char *)alloca(ext_str_size + 1);@@ -7496,6 +7514,7 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_tev->type == GGML_TYPE_F16 && op->type == GGML_TYPE_F16;const bool is_f32_f16 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F16 &&v->type == GGML_TYPE_F16 && op->type == GGML_TYPE_F32;+const bool is_f32_q8_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q8_0 &&v->type == GGML_TYPE_Q8_0 && op->type == GGML_TYPE_F32 &&dk % 32 == 0 && dv % 32 == 0;@@ -7503,6 +7522,21 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_tev->type == GGML_TYPE_Q4_0 && op->type == GGML_TYPE_F32 &&dk % 32 == 0 && dv % 32 == 0;+ // A7X (Adreno 740, compiler E031.41) SIGSEGVs inside clBuildProgram+ // building the flash_attn programs whose KV path is mixed-type or+ // dequantized — f32_f16, q8_0, q4_0 (reproduced at DK=40 and DK=64; it+ // is DK-independent). It is a driver crash, not codegen-wrong-output, so+ // it cannot be caught in-process (fatal=false only handles clean compile+ // errors). The uniform f16_f16 / f32_f32 programs compile fine on this+ // compiler, so decline only the KV-convert variants; ggml then runs+ // those (f16-KV / quant-KV) attention layers on the CPU backend.+ // Negative compiler carve-out, same idiom as the Intel DK=512 decline+ // below and the X1E driver-quirk guards.+ if (backend_ctx && backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X &&+ (is_f32_f16 || is_f32_q8_0 || is_f32_q4_0)) {+ return false;+ }+// Asymmetric KV: host-dequants both sides to F32, uses f32 kernel.auto is_kv_type_ok = [](ggml_type t) {return t == GGML_TYPE_F16 || t == GGML_TYPE_F32 ||@@ -20583,6 +20617,12 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, coCL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne1));CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &r2));CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &r3));+ // The optimizer-barrier arg exists only in the ADRENO_OLD_COMPILER build of+ // this kernel; conformant compilers get the original 17-arg signature.+ if (backend_ctx->q6_k_flat_old_compiler) {+ cl_uchar q6k_mask = 0xFF; // never 0xFE in prod; see the kernel note+ CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_uchar), &q6k_mask));+ }#elsekernel = backend_ctx->kernel_mul_mv_q6_K_f32;diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.clindex 57b90c05ae..2cca5335dd 100644--- a/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl+++ b/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl@@ -28,6 +28,13 @@#define QK_K 256+// ADRENO_OLD_COMPILER is defined by the host (-D) only for the Adreno E031+// compilers older than E031.45, which miscompile several constructs this kernel+// used (confirmed on E031.38 and E031.41; E031.45 is clean). Every other+// compiler -- newer E031, E17, DX, Intel, and every non-Adreno device that+// builds this program -- takes the #else branches, which are the original+// source: the workarounds below cost ~13% on the q6_K flat n=1 GEMV where they+// are not needed.inline float block_q_6_K_dot_y_flat(global uchar * blk_ql,global uchar * blk_qh,@@ -37,6 +44,9 @@ inline float block_q_6_K_dot_y_flat(int ip,int is,int l0,+#if defined(ADRENO_OLD_COMPILER)+ int dbg,+#endiffloat4 y0,float4 y1,float4 y2,@@ -48,10 +58,40 @@ inline float block_q_6_K_dot_y_flat(global uchar * q1 = blk_ql + ib*128 + q_offset_l;global uchar * q2 = q1 + QK_K/8;global uchar * qh = blk_qh + ib*64 + q_offset_h;- global char * sc = blk_scales + ib*16 + is;float dall = blk_d[ib];+#if defined(ADRENO_OLD_COMPILER)+ // The vectorized dequant (int4/float4 bit-ops, convert_*4, dot()) and vload4+ // are miscompiled here -> garbage weights. Reconstruct the 6-bit weights and+ // take the dot product scalar. q4_K/q5_K flat already use scalar paths, which+ // is why q6_K was the only flat GEMV that failed.+ // Scales are SIGNED int8; read as uchar and sign-extend arithmetically so the+ // result does not depend on whether the compiler treats `char` as signed.+ global uchar * sc = (global uchar *)(blk_scales + ib*16 + is);++ int s0 = (int)sc[0] - 256*(sc[0] >> 7);+ int s2 = (int)sc[2] - 256*(sc[2] >> 7);+ int s4 = (int)sc[4] - 256*(sc[4] >> 7);+ int s6 = (int)sc[6] - 256*(sc[6] >> 7);++ // one 6-bit weight: low/high nibble of a ql byte OR'd with a 2-bit qh plane+ // (plane p in {0,1,2,3} selects qh bits 2p..2p+1) placed at bits 4-5, minus 32.+ #define Q6W(qb, sh, hb, p) ((float)((((int)(qb) >> (sh)) & 15) | ((((int)(hb) >> (2*(p))) & 3) << 4)) - 32.f)++ float d0 = y0.s0*Q6W(q1[0],0,qh[0],0) + y0.s1*Q6W(q1[1],0,qh[1],0) + y0.s2*Q6W(q1[2],0,qh[2],0) + y0.s3*Q6W(q1[3],0,qh[3],0);+ float d1 = y1.s0*Q6W(q2[0],0,qh[0],1) + y1.s1*Q6W(q2[1],0,qh[1],1) + y1.s2*Q6W(q2[2],0,qh[2],1) + y1.s3*Q6W(q2[3],0,qh[3],1);+ float d2 = y2.s0*Q6W(q1[0],4,qh[0],2) + y2.s1*Q6W(q1[1],4,qh[1],2) + y2.s2*Q6W(q1[2],4,qh[2],2) + y2.s3*Q6W(q1[3],4,qh[3],2);+ float d3 = y3.s0*Q6W(q2[0],4,qh[0],3) + y3.s1*Q6W(q2[1],4,qh[1],3) + y3.s2*Q6W(q2[2],4,qh[2],3) + y3.s3*Q6W(q2[3],4,qh[3],3);+ #undef Q6W++ if (dbg) printf("HELPER dall=%f s=[%d %d %d %d] d=[%f %f %f %f] ql0=%d qh0=%d y00=%f\n",+ dall, s0, s2, s4, s6, d0, d1, d2, d3, (int)q1[0], (int)qh[0], y0.s0);++ return dall * (d0 * s0 + d1 * s2 + d2 * s4 + d3 * s6);+#else+ global char * sc = blk_scales + ib*16 + is;+// Vectorized loads: 3 uchar4 weight loads instead of 12 scalar byte reads.// q_offset_l/h are 4-aligned, so these are aligned vector loads.uchar4 q1v = vload4(0, q1);@@ -72,6 +112,7 @@ inline float block_q_6_K_dot_y_flat(return dall * (dot(y0, w0) * sc[0] + dot(y1, w1) * sc[2] +dot(y2, w2) * sc[4] + dot(y3, w3) * sc[6]);+#endif}#undef N_DST@@ -113,6 +154,11 @@ kernel void kernel_mul_mv_q6_K_f32_flat(int ne1,int r2,int r3+#if defined(ADRENO_OLD_COMPILER)+ ,+ uchar q6k_mask // runtime 0xFF; the host passes it so the compiler cannot+ // constant-fold the printf guards below into nothing+#endif) {src1 = (global float*)((global char*)src1 + offset1);dst = (global float*)((global char*)dst + offsetd);@@ -128,6 +174,22 @@ kernel void kernel_mul_mv_q6_K_f32_flat(int first_row = (N_SIMDGROUP * r0 + get_sub_group_id()) * N_DST;+#if defined(ADRENO_OLD_COMPILER)+ // 64-bit `ulong` integer arithmetic is miscompiled here -> the base-pointer byte+ // offsets came out wrong, so EVERY weight/scale read hit the wrong address. This+ // was the primary cause of the q6_K flat failure (q5_K uses int offsets and is+ // unaffected). Compute the block index in `int` and widen to `ulong` only inside+ // the pointer expression: the byte offset stays 64-bit, but there is no ulong+ // arithmetic chain to miscompile. The int index would overflow past ~2^31 blocks,+ // which no realistic weight reaches -- but that is a narrowing, so keep it off the+ // conformant path, which retains full ulong arithmetic.+ int offset_src0 = first_row*nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);++ global uchar * blk_ql = (global uchar *) src0_ql + (ulong)offset_src0 * 128;+ global uchar * blk_qh = (global uchar *) src0_qh + (ulong)offset_src0 * 64;+ global char * blk_scales = (global char *) src0_s + (ulong)offset_src0 * 16;+ global half * blk_d = (global half *) src0_d + offset_src0;+#elseulong offset_src0 = first_row*nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);ulong offset_src0_ql = offset_src0 * 128;ulong offset_src0_qh = offset_src0 * 64;@@ -138,6 +200,7 @@ kernel void kernel_mul_mv_q6_K_f32_flat(global uchar * blk_qh = (global uchar *) src0_qh + offset_src0_qh;global char * blk_scales = (global char *) src0_s + offset_src0_s;global half * blk_d = (global half *) src0_d + offset_src0_d;+#endifglobal float * yy = (global float *) src1 + r1*ne10 + im*ne00*ne1;int tid = get_sub_group_local_id()%(N_SIMDWIDTH/BLOCK_STRIDE); // within-super-block part, 0..15@@ -155,24 +218,55 @@ kernel void kernel_mul_mv_q6_K_f32_flat(for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {global float * y = yy + ib * QK_K + 128*ip + l0;+#if defined(ADRENO_OLD_COMPILER)+ // vload4 of f32 is miscompiled here; index the lanes scalar instead.+ float4 y0 = (float4)(y[ 0], y[ 1], y[ 2], y[ 3]);+ float4 y1 = (float4)(y[32], y[33], y[34], y[35]);+ float4 y2 = (float4)(y[64], y[65], y[66], y[67]);+ float4 y3 = (float4)(y[96], y[97], y[98], y[99]);+#elsefloat4 y0 = vload4(0, y + 0);float4 y1 = vload4(0, y + 32);float4 y2 = vload4(0, y + 64);float4 y3 = vload4(0, y + 96);+#endiffor (int row = 0; row < N_DST; row++) {if (first_row + row < ne01) {+#if defined(ADRENO_OLD_COMPILER)+ int dbg = (q6k_mask==0xFE && r0==0 && r1==0 && im==0 && row==0 && ib==0 &&+ ne00==256 && ne01==16 && get_sub_group_local_id()==0) ? 1 : 0;+ sumf[row] += block_q_6_K_dot_y_flat(+ blk_ql + row*nb*128, blk_qh + row*nb*64, blk_scales + row*nb*16, blk_d + row*nb,+ ib, ip, is, l0, dbg, y0, y1, y2, y3);+#elsesumf[row] += block_q_6_K_dot_y_flat(blk_ql + row*nb*128, blk_qh + row*nb*64, blk_scales + row*nb*16, blk_d + row*nb,ib, ip, is, l0, y0, y1, y2, y3);+#endif}}}+#if defined(ADRENO_OLD_COMPILER)+ // Optimizer barrier. This compiler drops the sumf partials unless a side effect+ // forces them to materialize. q6k_mask is a kernel arg the compiler cannot prove+ // is never 0xFE (the host always passes 0xFF), so the printf survives compilation+ // but never executes. FRAGILE: the exact set and placement of these guarded+ // printfs is load-bearing on E031.41 -- removing any one re-breaks q6_K.+ if (q6k_mask==0xFE && r0==0 && r1==0 && im==0 && ne00==256 && ne01==16 && get_sub_group_local_id()<16) {+ printf("Q6KLANE lane=%d ip=%d il=%d is=%d l0=%d sumf0=%f\n",+ get_sub_group_local_id(), ip, il, is, l0, sumf[0]);+ }+#endiffor (int row = 0; row < N_DST; row++) {float tot = sub_group_reduce_add(sumf[row]);if (get_sub_group_local_id() == 0 && first_row + row < ne01) {dst[r1*ne0 + im*ne0*ne1 + first_row + row] = tot;+#if defined(ADRENO_OLD_COMPILER)+ if (q6k_mask==0xFE && r0==0 && r1==0 && im==0 && row==0 && ne00==256 && ne01==16)+ printf("Q6KTOT tot=%f\n", tot);+#endif}}}