Skip to content

LLM inference in C/C++

C++ ggml

opencl: fix q6_K flat mul_mat for Adreno A6x/A7x GPUs with older E031 compilers (#26476)

Hongqiang Wang6503355df0eb
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp
index d169f33896..49cd9fd355 100644
--- a/ggml/src/ggml-opencl/ggml-opencl.cpp
+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp
@@ -582,6 +582,8 @@ struct ggml_backend_opencl_context {
bool adreno_use_bin_kernels;
get_adreno_bin_kernel_func_t get_adreno_bin_kernel_func = nullptr;
ggml_cl_compiler_version adreno_cl_compiler_version;
+ // The q6_K flat mul_mat codegen workarounds are needed by old E031 compilers only.
+ bool q6_k_flat_old_compiler;
std::string kernel_compile_opts; // cached for lazy-compiled kernels.
@@ -1931,8 +1933,14 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
#else
const std::string kernel_src = read_file("mul_mv_q6_k_f32_flat.cl");
#endif
+ // The codegen workarounds in this kernel are a measured 13-20% loss on
+ // compilers that do not need them, so only the affected ones build them;
+ // everyone else gets the original source.
+ const std::string q6k_opts = backend_ctx->q6_k_flat_old_compiler
+ ? compile_opts + " -DADRENO_OLD_COMPILER=1"
+ : compile_opts;
cl_program prog =
- build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
+ build_program_from_source(backend_ctx, kernel_src.c_str(), q6k_opts);
CL_CHECK((backend_ctx->kernel_mul_mv_q6_K_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q6_K_f32_flat", &err), err));
CL_CHECK(clReleaseProgram(prog));
@@ -5917,6 +5925,16 @@ static ggml_backend_opencl_context * ggml_cl_init(ggml_backend_dev_t dev) {
(backend_ctx->adreno_cl_compiler_version.type == E031 && backend_ctx->adreno_cl_compiler_version.major >= 47) ||
(backend_ctx->adreno_cl_compiler_version.type == DX && backend_ctx->adreno_cl_compiler_version.major >= 17);
+ // The q6_K flat mul_mat miscompile is a defect of the older E031 compilers, not a
+ // property of any GPU generation: it reproduces on E031.38 (Adreno 642L) and E031.41
+ // (Adreno 740) and is fixed by E031.45 (Adreno 619). Gate on the compiler so parts
+ // that do not need the workarounds do not pay for them. The explicit type check is
+ // required: newer_than_or_same() is false for every non-E031 compiler, so negating it
+ // alone would enable the workarounds on E17/DX.
+ backend_ctx->q6_k_flat_old_compiler =
+ backend_ctx->adreno_cl_compiler_version.type == E031 &&
+ !backend_ctx->adreno_cl_compiler_version.newer_than_or_same(E031, 45, 0, 0);
+
size_t ext_str_size;
clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, 0, NULL, &ext_str_size);
char *ext_buffer = (char *)alloca(ext_str_size + 1);
@@ -7496,6 +7514,7 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
v->type == GGML_TYPE_F16 && op->type == GGML_TYPE_F16;
const bool is_f32_f16 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F16 &&
v->type == GGML_TYPE_F16 && op->type == GGML_TYPE_F32;
+
const bool is_f32_q8_0 = q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_Q8_0 &&
v->type == GGML_TYPE_Q8_0 && op->type == GGML_TYPE_F32 &&
dk % 32 == 0 && dv % 32 == 0;
@@ -7503,6 +7522,21 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
v->type == GGML_TYPE_Q4_0 && op->type == GGML_TYPE_F32 &&
dk % 32 == 0 && dv % 32 == 0;
+ // A7X (Adreno 740, compiler E031.41) SIGSEGVs inside clBuildProgram
+ // building the flash_attn programs whose KV path is mixed-type or
+ // dequantized — f32_f16, q8_0, q4_0 (reproduced at DK=40 and DK=64; it
+ // is DK-independent). It is a driver crash, not codegen-wrong-output, so
+ // it cannot be caught in-process (fatal=false only handles clean compile
+ // errors). The uniform f16_f16 / f32_f32 programs compile fine on this
+ // compiler, so decline only the KV-convert variants; ggml then runs
+ // those (f16-KV / quant-KV) attention layers on the CPU backend.
+ // Negative compiler carve-out, same idiom as the Intel DK=512 decline
+ // below and the X1E driver-quirk guards.
+ if (backend_ctx && backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X &&
+ (is_f32_f16 || is_f32_q8_0 || is_f32_q4_0)) {
+ return false;
+ }
+
// Asymmetric KV: host-dequants both sides to F32, uses f32 kernel.
auto is_kv_type_ok = [](ggml_type t) {
return t == GGML_TYPE_F16 || t == GGML_TYPE_F32 ||
@@ -20583,6 +20617,12 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne1));
CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &r2));
CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &r3));
+ // The optimizer-barrier arg exists only in the ADRENO_OLD_COMPILER build of
+ // this kernel; conformant compilers get the original 17-arg signature.
+ if (backend_ctx->q6_k_flat_old_compiler) {
+ cl_uchar q6k_mask = 0xFF; // never 0xFE in prod; see the kernel note
+ CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_uchar), &q6k_mask));
+ }
#else
kernel = backend_ctx->kernel_mul_mv_q6_K_f32;
diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl
index 57b90c05ae..2cca5335dd 100644
--- a/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl
+++ b/ggml/src/ggml-opencl/kernels/mul_mv_q6_k_f32_flat.cl
@@ -28,6 +28,13 @@
#define QK_K 256
+// ADRENO_OLD_COMPILER is defined by the host (-D) only for the Adreno E031
+// compilers older than E031.45, which miscompile several constructs this kernel
+// used (confirmed on E031.38 and E031.41; E031.45 is clean). Every other
+// compiler -- newer E031, E17, DX, Intel, and every non-Adreno device that
+// builds this program -- takes the #else branches, which are the original
+// source: the workarounds below cost ~13% on the q6_K flat n=1 GEMV where they
+// are not needed.
inline float block_q_6_K_dot_y_flat(
global uchar * blk_ql,
global uchar * blk_qh,
@@ -37,6 +44,9 @@ inline float block_q_6_K_dot_y_flat(
int ip,
int is,
int l0,
+#if defined(ADRENO_OLD_COMPILER)
+ int dbg,
+#endif
float4 y0,
float4 y1,
float4 y2,
@@ -48,10 +58,40 @@ inline float block_q_6_K_dot_y_flat(
global uchar * q1 = blk_ql + ib*128 + q_offset_l;
global uchar * q2 = q1 + QK_K/8;
global uchar * qh = blk_qh + ib*64 + q_offset_h;
- global char * sc = blk_scales + ib*16 + is;
float dall = blk_d[ib];
+#if defined(ADRENO_OLD_COMPILER)
+ // The vectorized dequant (int4/float4 bit-ops, convert_*4, dot()) and vload4
+ // are miscompiled here -> garbage weights. Reconstruct the 6-bit weights and
+ // take the dot product scalar. q4_K/q5_K flat already use scalar paths, which
+ // is why q6_K was the only flat GEMV that failed.
+ // Scales are SIGNED int8; read as uchar and sign-extend arithmetically so the
+ // result does not depend on whether the compiler treats `char` as signed.
+ global uchar * sc = (global uchar *)(blk_scales + ib*16 + is);
+
+ int s0 = (int)sc[0] - 256*(sc[0] >> 7);
+ int s2 = (int)sc[2] - 256*(sc[2] >> 7);
+ int s4 = (int)sc[4] - 256*(sc[4] >> 7);
+ int s6 = (int)sc[6] - 256*(sc[6] >> 7);
+
+ // one 6-bit weight: low/high nibble of a ql byte OR'd with a 2-bit qh plane
+ // (plane p in {0,1,2,3} selects qh bits 2p..2p+1) placed at bits 4-5, minus 32.
+ #define Q6W(qb, sh, hb, p) ((float)((((int)(qb) >> (sh)) & 15) | ((((int)(hb) >> (2*(p))) & 3) << 4)) - 32.f)
+
+ float d0 = y0.s0*Q6W(q1[0],0,qh[0],0) + y0.s1*Q6W(q1[1],0,qh[1],0) + y0.s2*Q6W(q1[2],0,qh[2],0) + y0.s3*Q6W(q1[3],0,qh[3],0);
+ float d1 = y1.s0*Q6W(q2[0],0,qh[0],1) + y1.s1*Q6W(q2[1],0,qh[1],1) + y1.s2*Q6W(q2[2],0,qh[2],1) + y1.s3*Q6W(q2[3],0,qh[3],1);
+ float d2 = y2.s0*Q6W(q1[0],4,qh[0],2) + y2.s1*Q6W(q1[1],4,qh[1],2) + y2.s2*Q6W(q1[2],4,qh[2],2) + y2.s3*Q6W(q1[3],4,qh[3],2);
+ float d3 = y3.s0*Q6W(q2[0],4,qh[0],3) + y3.s1*Q6W(q2[1],4,qh[1],3) + y3.s2*Q6W(q2[2],4,qh[2],3) + y3.s3*Q6W(q2[3],4,qh[3],3);
+ #undef Q6W
+
+ if (dbg) printf("HELPER dall=%f s=[%d %d %d %d] d=[%f %f %f %f] ql0=%d qh0=%d y00=%f\n",
+ dall, s0, s2, s4, s6, d0, d1, d2, d3, (int)q1[0], (int)qh[0], y0.s0);
+
+ return dall * (d0 * s0 + d1 * s2 + d2 * s4 + d3 * s6);
+#else
+ global char * sc = blk_scales + ib*16 + is;
+
// Vectorized loads: 3 uchar4 weight loads instead of 12 scalar byte reads.
// q_offset_l/h are 4-aligned, so these are aligned vector loads.
uchar4 q1v = vload4(0, q1);
@@ -72,6 +112,7 @@ inline float block_q_6_K_dot_y_flat(
return dall * (dot(y0, w0) * sc[0] + dot(y1, w1) * sc[2] +
dot(y2, w2) * sc[4] + dot(y3, w3) * sc[6]);
+#endif
}
#undef N_DST
@@ -113,6 +154,11 @@ kernel void kernel_mul_mv_q6_K_f32_flat(
int ne1,
int r2,
int r3
+#if defined(ADRENO_OLD_COMPILER)
+ ,
+ uchar q6k_mask // runtime 0xFF; the host passes it so the compiler cannot
+ // constant-fold the printf guards below into nothing
+#endif
) {
src1 = (global float*)((global char*)src1 + offset1);
dst = (global float*)((global char*)dst + offsetd);
@@ -128,6 +174,22 @@ kernel void kernel_mul_mv_q6_K_f32_flat(
int first_row = (N_SIMDGROUP * r0 + get_sub_group_id()) * N_DST;
+#if defined(ADRENO_OLD_COMPILER)
+ // 64-bit `ulong` integer arithmetic is miscompiled here -> the base-pointer byte
+ // offsets came out wrong, so EVERY weight/scale read hit the wrong address. This
+ // was the primary cause of the q6_K flat failure (q5_K uses int offsets and is
+ // unaffected). Compute the block index in `int` and widen to `ulong` only inside
+ // the pointer expression: the byte offset stays 64-bit, but there is no ulong
+ // arithmetic chain to miscompile. The int index would overflow past ~2^31 blocks,
+ // which no realistic weight reaches -- but that is a narrowing, so keep it off the
+ // conformant path, which retains full ulong arithmetic.
+ int offset_src0 = first_row*nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
+
+ global uchar * blk_ql = (global uchar *) src0_ql + (ulong)offset_src0 * 128;
+ global uchar * blk_qh = (global uchar *) src0_qh + (ulong)offset_src0 * 64;
+ global char * blk_scales = (global char *) src0_s + (ulong)offset_src0 * 16;
+ global half * blk_d = (global half *) src0_d + offset_src0;
+#else
ulong offset_src0 = first_row*nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
ulong offset_src0_ql = offset_src0 * 128;
ulong offset_src0_qh = offset_src0 * 64;
@@ -138,6 +200,7 @@ kernel void kernel_mul_mv_q6_K_f32_flat(
global uchar * blk_qh = (global uchar *) src0_qh + offset_src0_qh;
global char * blk_scales = (global char *) src0_s + offset_src0_s;
global half * blk_d = (global half *) src0_d + offset_src0_d;
+#endif
global float * yy = (global float *) src1 + r1*ne10 + im*ne00*ne1;
int tid = get_sub_group_local_id()%(N_SIMDWIDTH/BLOCK_STRIDE); // within-super-block part, 0..15
@@ -155,24 +218,55 @@ kernel void kernel_mul_mv_q6_K_f32_flat(
for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) {
global float * y = yy + ib * QK_K + 128*ip + l0;
+#if defined(ADRENO_OLD_COMPILER)
+ // vload4 of f32 is miscompiled here; index the lanes scalar instead.
+ float4 y0 = (float4)(y[ 0], y[ 1], y[ 2], y[ 3]);
+ float4 y1 = (float4)(y[32], y[33], y[34], y[35]);
+ float4 y2 = (float4)(y[64], y[65], y[66], y[67]);
+ float4 y3 = (float4)(y[96], y[97], y[98], y[99]);
+#else
float4 y0 = vload4(0, y + 0);
float4 y1 = vload4(0, y + 32);
float4 y2 = vload4(0, y + 64);
float4 y3 = vload4(0, y + 96);
+#endif
for (int row = 0; row < N_DST; row++) {
if (first_row + row < ne01) {
+#if defined(ADRENO_OLD_COMPILER)
+ int dbg = (q6k_mask==0xFE && r0==0 && r1==0 && im==0 && row==0 && ib==0 &&
+ ne00==256 && ne01==16 && get_sub_group_local_id()==0) ? 1 : 0;
+ sumf[row] += block_q_6_K_dot_y_flat(
+ blk_ql + row*nb*128, blk_qh + row*nb*64, blk_scales + row*nb*16, blk_d + row*nb,
+ ib, ip, is, l0, dbg, y0, y1, y2, y3);
+#else
sumf[row] += block_q_6_K_dot_y_flat(
blk_ql + row*nb*128, blk_qh + row*nb*64, blk_scales + row*nb*16, blk_d + row*nb,
ib, ip, is, l0, y0, y1, y2, y3);
+#endif
}
}
}
+#if defined(ADRENO_OLD_COMPILER)
+ // Optimizer barrier. This compiler drops the sumf partials unless a side effect
+ // forces them to materialize. q6k_mask is a kernel arg the compiler cannot prove
+ // is never 0xFE (the host always passes 0xFF), so the printf survives compilation
+ // but never executes. FRAGILE: the exact set and placement of these guarded
+ // printfs is load-bearing on E031.41 -- removing any one re-breaks q6_K.
+ if (q6k_mask==0xFE && r0==0 && r1==0 && im==0 && ne00==256 && ne01==16 && get_sub_group_local_id()<16) {
+ printf("Q6KLANE lane=%d ip=%d il=%d is=%d l0=%d sumf0=%f\n",
+ get_sub_group_local_id(), ip, il, is, l0, sumf[0]);
+ }
+#endif
for (int row = 0; row < N_DST; row++) {
float tot = sub_group_reduce_add(sumf[row]);
if (get_sub_group_local_id() == 0 && first_row + row < ne01) {
dst[r1*ne0 + im*ne0*ne1 + first_row + row] = tot;
+#if defined(ADRENO_OLD_COMPILER)
+ if (q6k_mask==0xFE && r0==0 && r1==0 && im==0 && row==0 && ne00==256 && ne01==16)
+ printf("Q6KTOT tot=%f\n", tot);
+#endif
}
}
}