LLM inference in C/C++
C++ ggml
opencl: loads quants as uint for q4_K and q5_K flat mv (optimization for Adreno A7x GPUs) (#25780)
Hongqiang Wange8f19cc0ad70
diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.clindex d92fb96890..70391866ca 100644--- a/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl+++ b/ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl@@ -153,18 +153,27 @@ kernel void kernel_mul_mv_q4_K_f32_flat(global ushort * q2 = q1 + 32;- float4 acc1 = {0.f, 0.f, 0.f, 0.f};- float4 acc2 = {0.f, 0.f, 0.f, 0.f};- for (int i = 0; i < 8; i += 2) {- acc1.s0 += yl[i+0] * (q1[i/2] & 0x000F);- acc1.s1 += yl[i+1] * (q1[i/2] & 0x0F00);- acc1.s2 += yl[i+8] * (q1[i/2] & 0x00F0);- acc1.s3 += yl[i+9] * (q1[i/2] & 0xF000);- acc2.s0 += yh[i+0] * (q2[i/2] & 0x000F);- acc2.s1 += yh[i+1] * (q2[i/2] & 0x0F00);- acc2.s2 += yh[i+8] * (q2[i/2] & 0x00F0);- acc2.s3 += yh[i+9] * (q2[i/2] & 0xF000);- }+ // Load the 4 q1 / 4 q2 quant ushorts as 2 uints each. 16-bit integer ops are+ // disproportionately slow on the A7X (E031.41) compiler; keeping the dequant+ // operands in 32-bit registers avoids the ushort path. q1/q2 are 4-byte aligned+ // (ib*128 + (32*iq+8*ir) bytes; q1 += blk*128 bytes/row). Math is unchanged:+ // w & 0x0F00 on the low/high halves equals the original ushort mask value.+ global uint * q1u = (global uint *)q1;+ global uint * q2u = (global uint *)q2;+ uint a0 = q1u[0], a1 = q1u[1];+ uint b0 = q2u[0], b1 = q2u[1];+ uint w0 = a0 & 0xFFFF, w1 = a0 >> 16, w2 = a1 & 0xFFFF, w3 = a1 >> 16;+ uint v0 = b0 & 0xFFFF, v1 = b0 >> 16, v2 = b1 & 0xFFFF, v3 = b1 >> 16;++ float4 acc1, acc2;+ acc1.s0 = yl[0]*(w0&0x000F) + yl[ 2]*(w1&0x000F) + yl[ 4]*(w2&0x000F) + yl[ 6]*(w3&0x000F);+ acc1.s1 = yl[1]*(w0&0x0F00) + yl[ 3]*(w1&0x0F00) + yl[ 5]*(w2&0x0F00) + yl[ 7]*(w3&0x0F00);+ acc1.s2 = yl[8]*(w0&0x00F0) + yl[10]*(w1&0x00F0) + yl[12]*(w2&0x00F0) + yl[14]*(w3&0x00F0);+ acc1.s3 = yl[9]*(w0&0xF000) + yl[11]*(w1&0xF000) + yl[13]*(w2&0xF000) + yl[15]*(w3&0xF000);+ acc2.s0 = yh[0]*(v0&0x000F) + yh[ 2]*(v1&0x000F) + yh[ 4]*(v2&0x000F) + yh[ 6]*(v3&0x000F);+ acc2.s1 = yh[1]*(v0&0x0F00) + yh[ 3]*(v1&0x0F00) + yh[ 5]*(v2&0x0F00) + yh[ 7]*(v3&0x0F00);+ acc2.s2 = yh[8]*(v0&0x00F0) + yh[10]*(v1&0x00F0) + yh[12]*(v2&0x00F0) + yh[14]*(v3&0x00F0);+ acc2.s3 = yh[9]*(v0&0xF000) + yh[11]*(v1&0xF000) + yh[13]*(v2&0xF000) + yh[15]*(v3&0xF000);float dall = *d;float dmin = *dm;diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.clindex e353a72be7..6020364b5c 100644--- a/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl+++ b/ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl@@ -159,18 +159,59 @@ kernel void kernel_mul_mv_q5_K_f32_flat(global ushort * q2 = q1 + 32;- float4 acc1 = {0.f, 0.f, 0.f, 0.f};- float4 acc2 = {0.f, 0.f, 0.f, 0.f};- for (int i = 0; i < 8; i += 2) {- acc1.s0 += yl[i+0] * ((q1[i/2] & 0x000F) + (qh[i+0] & u1_lo ? 16.f : 0.f));- acc1.s1 += yl[i+1] * ((q1[i/2] & 0x0F00) + (qh[i+1] & u1_lo ? 16.f*256.f : 0.f));- acc1.s2 += yl[i+8] * ((q1[i/2] & 0x00F0) + (qh[i+0] & u2_lo ? 16.f*16.f : 0.f));- acc1.s3 += yl[i+9] * ((q1[i/2] & 0xF000) + (qh[i+1] & u2_lo ? 16.f*4096.f: 0.f));- acc2.s0 += yh[i+0] * ((q2[i/2] & 0x000F) + (qh[i+0] & u1_hi ? 16.f : 0.f));- acc2.s1 += yh[i+1] * ((q2[i/2] & 0x0F00) + (qh[i+1] & u1_hi ? 16.f*256.f : 0.f));- acc2.s2 += yh[i+8] * ((q2[i/2] & 0x00F0) + (qh[i+0] & u2_hi ? 16.f*16.f : 0.f));- acc2.s3 += yh[i+9] * ((q2[i/2] & 0xF000) + (qh[i+1] & u2_hi ? 16.f*4096.f: 0.f));- }+ // Load the 4 q1 / 4 q2 quant ushorts as 2 uints each. 16-bit integer ops are+ // disproportionately slow on the A7X (E031.41) compiler; keeping the dequant+ // operands in 32-bit registers avoids the ushort path (same fix as q4_K flat).+ // q1/q2 are 4-byte aligned; w & 0x0F00 on the low/high half of a uint equals the+ // original ushort mask value, so the math is unchanged. The qh high-bit term is+ // byte-indexed (qh[0..7]) and left as-is.+ global uint * q1u = (global uint *)q1;+ global uint * q2u = (global uint *)q2;+ uint a0 = q1u[0], a1 = q1u[1], b0 = q2u[0], b1 = q2u[1];+ uint w0 = a0 & 0xFFFF, w1 = a0 >> 16, w2 = a1 & 0xFFFF, w3 = a1 >> 16;+ uint v0 = b0 & 0xFFFF, v1 = b0 >> 16, v2 = b1 & 0xFFFF, v3 = b1 >> 16;++ float4 acc1, acc2;+ acc1.s0 =+ yl[0]*((w0&0x000F)+(qh[0]&u1_lo?16.f:0.f)) ++ yl[2]*((w1&0x000F)+(qh[2]&u1_lo?16.f:0.f)) ++ yl[4]*((w2&0x000F)+(qh[4]&u1_lo?16.f:0.f)) ++ yl[6]*((w3&0x000F)+(qh[6]&u1_lo?16.f:0.f));+ acc1.s1 =+ yl[1]*((w0&0x0F00)+(qh[1]&u1_lo?16.f*256.f:0.f)) ++ yl[3]*((w1&0x0F00)+(qh[3]&u1_lo?16.f*256.f:0.f)) ++ yl[5]*((w2&0x0F00)+(qh[5]&u1_lo?16.f*256.f:0.f)) ++ yl[7]*((w3&0x0F00)+(qh[7]&u1_lo?16.f*256.f:0.f));+ acc1.s2 =+ yl[ 8]*((w0&0x00F0)+(qh[0]&u2_lo?16.f*16.f:0.f)) ++ yl[10]*((w1&0x00F0)+(qh[2]&u2_lo?16.f*16.f:0.f)) ++ yl[12]*((w2&0x00F0)+(qh[4]&u2_lo?16.f*16.f:0.f)) ++ yl[14]*((w3&0x00F0)+(qh[6]&u2_lo?16.f*16.f:0.f));+ acc1.s3 =+ yl[ 9]*((w0&0xF000)+(qh[1]&u2_lo?16.f*4096.f:0.f)) ++ yl[11]*((w1&0xF000)+(qh[3]&u2_lo?16.f*4096.f:0.f)) ++ yl[13]*((w2&0xF000)+(qh[5]&u2_lo?16.f*4096.f:0.f)) ++ yl[15]*((w3&0xF000)+(qh[7]&u2_lo?16.f*4096.f:0.f));+ acc2.s0 =+ yh[0]*((v0&0x000F)+(qh[0]&u1_hi?16.f:0.f)) ++ yh[2]*((v1&0x000F)+(qh[2]&u1_hi?16.f:0.f)) ++ yh[4]*((v2&0x000F)+(qh[4]&u1_hi?16.f:0.f)) ++ yh[6]*((v3&0x000F)+(qh[6]&u1_hi?16.f:0.f));+ acc2.s1 =+ yh[1]*((v0&0x0F00)+(qh[1]&u1_hi?16.f*256.f:0.f)) ++ yh[3]*((v1&0x0F00)+(qh[3]&u1_hi?16.f*256.f:0.f)) ++ yh[5]*((v2&0x0F00)+(qh[5]&u1_hi?16.f*256.f:0.f)) ++ yh[7]*((v3&0x0F00)+(qh[7]&u1_hi?16.f*256.f:0.f));+ acc2.s2 =+ yh[ 8]*((v0&0x00F0)+(qh[0]&u2_hi?16.f*16.f:0.f)) ++ yh[10]*((v1&0x00F0)+(qh[2]&u2_hi?16.f*16.f:0.f)) ++ yh[12]*((v2&0x00F0)+(qh[4]&u2_hi?16.f*16.f:0.f)) ++ yh[14]*((v3&0x00F0)+(qh[6]&u2_hi?16.f*16.f:0.f));+ acc2.s3 =+ yh[ 9]*((v0&0xF000)+(qh[1]&u2_hi?16.f*4096.f:0.f)) ++ yh[11]*((v1&0xF000)+(qh[3]&u2_hi?16.f*4096.f:0.f)) ++ yh[13]*((v2&0xF000)+(qh[5]&u2_hi?16.f*4096.f:0.f)) ++ yh[15]*((v3&0xF000)+(qh[7]&u2_hi?16.f*4096.f:0.f));float dall = *d;float dmin = *dm;