LLM inference in C/C++
C++ ggml
opencl: add dp4a GEMM variants to vector load activation (#30753)
lhezb531e4c7494e
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cppindex 70f199dba1..6071b6e582 100644--- a/ggml/src/ggml-opencl/ggml-opencl.cpp+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp@@ -1243,6 +1243,7 @@ struct ggml_backend_opencl_context {cl_kernel kernel_gemm_noshuffle_q4_1_f32;cl_kernel kernel_gemm_noshuffle_q8_0_f32, kernel_gemm_noshuffle_q8_0_f32_bin;cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a = nullptr; // dp4a (int8) dense q8_0 prefill GEMM (opt-in)+ cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a_act_vload4 = nullptr;cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg = nullptr; // q8_0 dense dp4a, weights via texture (opt-in)cl_kernel kernel_gemm_noshuffle_q8_0_q8_1_dp4a_ila_a8_bin = nullptr;cl_kernel kernel_gemv_noshuffle_q8_0_f32;@@ -1264,6 +1265,7 @@ struct ggml_backend_opencl_context {cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin;cl_kernel kernel_gemv_noshuffle_q4_k_f32_32b_trans;cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a = nullptr; // dp4a (int8) dense prefill GEMM+ cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_act_vload4 = nullptr;cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg = nullptr; // dp4a dense prefill GEMM, weights via texture (X1 opt-in)cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_narrow = nullptr;cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_narrow_wimg = nullptr;@@ -1271,7 +1273,9 @@ struct ggml_backend_opencl_context {int tile_n_dp4a_narrow_q4_k = 32;int max_n_dp4a_narrow_q4_k = 16;cl_kernel kernel_gemm_noshuffle_q5_k_q8_1_dp4a = nullptr; // dp4a (int8) dense q5_K prefill GEMM+ cl_kernel kernel_gemm_noshuffle_q5_k_q8_1_dp4a_act_vload4 = nullptr;cl_kernel kernel_gemm_noshuffle_q6_k_q8_1_dp4a = nullptr; // dp4a (int8) dense q6_K prefill GEMM+ cl_kernel kernel_gemm_noshuffle_q6_k_q8_1_dp4a_act_vload4 = nullptr;cl_kernel kernel_quant_a_q8_1; // plain activation q8_1 pre-passcl_kernel kernel_gemm_noshuffle_q4_k_f32_r1;cl_kernel kernel_gemm_noshuffle_q4_k_f32_kimg;@@ -1305,6 +1309,7 @@ struct ggml_backend_opencl_context {cl_kernel kernel_gemm_noshuffle_iq4_nl_f32;cl_kernel kernel_gemm_noshuffle_iq4_nl_q8_1_dp4a = nullptr; // dp4a (int8) dense IQ4_NL prefill GEMMcl_kernel kernel_gemm_noshuffle_q4_0_q8_1_dp4a = nullptr; // dp4a (int8) dense q4_0 prefill GEMM+ cl_kernel kernel_gemm_noshuffle_q4_0_q8_1_dp4a_act_vload4 = nullptr;cl_kernel kernel_gemm_coopk_q4_0_q8_1_dp4a_c4 = nullptr;cl_kernel kernel_gemm_coopk_q4_0_q8_1_dp4a_c2 = nullptr;int nsg_coopk_dp4a_q4_0 = 8;@@ -4347,6 +4352,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {#endifcl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_0_q8_1_dp4a", &err), err));+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a_act_vload4 = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_0_q8_1_dp4a_act_vload4", &err), err));CL_CHECK(clReleaseProgram(prog));GGML_LOG_CONT(".");}@@ -4486,6 +4492,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), dp4a_opts);CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a", &err), err));CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg", &err), err));+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_act_vload4 = clCreateKernel(prog, "kernel_gemm_noshuffle_q4_k_q8_1_dp4a_act_vload4", &err), err));CL_CHECK(clReleaseProgram(prog));// Compile a smaller N tile to avoid unused columns in small batches.@@ -4518,6 +4525,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_q8_1_dp4a", &err), err));CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg", &err), err));+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_act_vload4 = clCreateKernel(prog, "kernel_gemm_noshuffle_q8_0_q8_1_dp4a_act_vload4", &err), err));CL_CHECK(clReleaseProgram(prog));GGML_LOG_CONT(".");}@@ -4533,6 +4541,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {#endifcl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_k_q8_1_dp4a", &err), err));+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a_act_vload4 = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_k_q8_1_dp4a_act_vload4", &err), err));CL_CHECK(clReleaseProgram(prog));GGML_LOG_CONT(".");}@@ -4548,6 +4557,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {#endifcl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_q6_k_q8_1_dp4a", &err), err));+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_act_vload4 = clCreateKernel(prog, "kernel_gemm_noshuffle_q6_k_q8_1_dp4a_act_vload4", &err), err));CL_CHECK(clReleaseProgram(prog));GGML_LOG_CONT(".");}@@ -20430,11 +20440,13 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_treturn;}- // dp4a (int8) dense prefill GEMM, default off+ static const char * q4_0_act_vload4_env = getenv("GGML_OPENCL_Q4_0_DP4A_ACT_VLOAD4");+ const bool q4_0_act_vload4_on = q4_0_act_vload4_env == nullptr || atoi(q4_0_act_vload4_env) != 0;+ // Enable dense prefill by default on X2E when the uint4 activation tile is enabled.static const char * q4_0_dense_dp4a_env = getenv("GGML_OPENCL_Q4_0_DENSE_DP4A");bool q4_0_dense_dp4a_on = q4_0_dense_dp4a_env? (atoi(q4_0_dense_dp4a_env) != 0)- : false;+ : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && q4_0_act_vload4_on);// dot prod has to be availableq4_0_dense_dp4a_on = backend_ctx->has_integer_dot && q4_0_dense_dp4a_on;@@ -20461,7 +20473,9 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_tsize_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) };backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst);- cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a;+ cl_kernel dk = q4_0_act_vload4_on+ ? backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a_act_vload4+ : backend_ctx->kernel_gemm_noshuffle_q4_0_q8_1_dp4a;int ai = 0;CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_0->q));CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q4_0->d));@@ -21754,8 +21768,13 @@ static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_t}}- cl_kernel dk = use_wimg ? backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg- : backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a;+ static const char * q8_0_act_vload4_env = getenv("GGML_OPENCL_Q8_0_DP4A_ACT_VLOAD4");+ const bool q8_0_act_vload4_on = q8_0_act_vload4_env == nullptr || atoi(q8_0_act_vload4_env) != 0;+ cl_kernel dk = use_wimg+ ? backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg+ : q8_0_act_vload4_on+ ? backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a_act_vload4+ : backend_ctx->kernel_gemm_noshuffle_q8_0_q8_1_dp4a;int ai = 0;if (use_wimg) {CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &q8_q_img));@@ -22703,11 +22722,16 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno(ggml_backend_t backend, const ggml_tconst bool use_narrow = backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_narrow != nullptr&& N <= backend_ctx->max_n_dp4a_narrow_q4_k;+ static const char * q4_k_act_vload4_env = getenv("GGML_OPENCL_Q4_K_DP4A_ACT_VLOAD4");+ const bool q4_k_act_vload4_on = q4_k_act_vload4_env == nullptr || atoi(q4_k_act_vload4_env) != 0;cl_kernel dk = use_narrow? (use_wimg ? backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_narrow_wimg: backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_narrow)- : (use_wimg ? backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg- : backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a);+ : use_wimg+ ? backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg+ : q4_k_act_vload4_on+ ? backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_act_vload4+ : backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a;int ai = 0;if (use_wimg) {CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &q4k_q_img));@@ -23369,7 +23393,11 @@ static void ggml_cl_mul_mat_q6_K_f32_adreno(ggml_backend_t backend, const ggml_tsize_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) };backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst);- cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a;+ static const char * q6_k_act_vload4_env = getenv("GGML_OPENCL_Q6_K_DP4A_ACT_VLOAD4");+ const bool q6_k_act_vload4_on = q6_k_act_vload4_env == nullptr || atoi(q6_k_act_vload4_env) != 0;+ cl_kernel dk = q6_k_act_vload4_on+ ? backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a_act_vload4+ : backend_ctx->kernel_gemm_noshuffle_q6_k_q8_1_dp4a;int ai = 0;CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q6_K->ql));CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q6_K->qh));@@ -23925,7 +23953,11 @@ static void ggml_cl_mul_mat_q5_K_f32_adreno(ggml_backend_t backend, const ggml_tsize_t q_global[1] = { (size_t)(((n_blocks + 63) / 64) * 64) };backend_ctx->enqueue_ndrange_kernel(qk, 1, q_global, q_local, dst);- cl_kernel dk = backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a;+ static const char * q5_k_act_vload4_env = getenv("GGML_OPENCL_Q5_K_DP4A_ACT_VLOAD4");+ const bool q5_k_act_vload4_on = q5_k_act_vload4_env == nullptr || atoi(q5_k_act_vload4_env) != 0;+ cl_kernel dk = q5_k_act_vload4_on+ ? backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a_act_vload4+ : backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a;int ai = 0;CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->q));CL_CHECK(clSetKernelArg(dk, ai++, sizeof(cl_mem), &extra0_q5_k->qh));diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.clindex 446a8eb184..6418d0f8ff 100644--- a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_0_q8_1_dp4a.cl@@ -125,3 +125,129 @@ kernel void kernel_gemm_noshuffle_q4_0_q8_1_dp4a(}#undef NGROUPS}++// Stage each 32-K activation block as two uint4 values.+inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) {+ int r = 0;+ r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r);+ return r;+}++__attribute__((qcom_wave_pair_mode(1)))+kernel void kernel_gemm_noshuffle_q4_0_q8_1_dp4a_act_vload4(+ __global const ushort * src0_q,+ __global const half * src0_d,+ __global const uint * src1_qa,+ __global const half * src1_da,+ __global const half * src1_sa,+ __global float * dst,+ ulong offsetd,+ int m,+ int n_no_padding,+ int k+) {+ dst = (global float *)((global char *)dst + offsetd);++ const uint lid = get_local_id(0);+ const uint block_id_m = get_global_id(1);+ const uint block_id_n = get_global_id(2);++ const uint row = block_id_m * 64 + lid;+ const uint col_base = block_id_n * TILESIZE_N;+ const bool row_valid = row < (uint)m;+ const uint rrow = row_valid ? row : 0;++ const uint k_u = (uint)k >> 2;+ const uint k_b = (uint)k >> 5;++ __local uint4 sh_qa4[TILESIZE_N][2];+ __local half sh_d[TILESIZE_N];+ __local half sh_s[TILESIZE_N];++#define NGROUPS (TILESIZE_N / 4)+ float4 acc[NGROUPS];+ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) {+ acc[g] = (float4)(0.0f);+ }++ for (uint step = 0; step < (uint)k; step += 32) {+ const uint sub = step >> 5;++ const float d_w = (float)src0_d[rrow + sub * (uint)m];++ const uint qsbase = rrow + (step >> 2) * (uint)m;+ uint8 qw;+ qw.s0 = EXP4(src0_q[qsbase + 0 * m]);+ qw.s1 = EXP4(src0_q[qsbase + 1 * m]);+ qw.s2 = EXP4(src0_q[qsbase + 2 * m]);+ qw.s3 = EXP4(src0_q[qsbase + 3 * m]);+ qw.s4 = EXP4(src0_q[qsbase + 4 * m]);+ qw.s5 = EXP4(src0_q[qsbase + 5 * m]);+ qw.s6 = EXP4(src0_q[qsbase + 6 * m]);+ qw.s7 = EXP4(src0_q[qsbase + 7 * m]);++ for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) {+ const uint t = idx >> 1;+ const uint v = idx & 1;+ const uint c = col_base + t;+ sh_qa4[t][v] = (c < (uint)n_no_padding)+ ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2))+ : (uint4)(0u);+ }+ if (lid < TILESIZE_N) {+ const uint c = col_base + lid;+ sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;+ sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0;+ }+ barrier(CLK_LOCAL_MEM_FENCE);++#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))+ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) {+ const int b = g * 4;+ float4 rf;+#define DOTV(T) dot8_q8a_v(qw, sh_qa4[T][0], sh_qa4[T][1])+ rf.s0 = (float)DOTV(b+0);+ rf.s1 = (float)DOTV(b+1);+ rf.s2 = (float)DOTV(b+2);+ rf.s3 = (float)DOTV(b+3);+#undef DOTV++ acc[g] += d_w * (LD4(sh_d, b) * rf - 8.0f * LD4(sh_s, b));+ }+#undef LD4+ barrier(CLK_LOCAL_MEM_FENCE);+ }++ if (!row_valid) {+ return;+ }++ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) {+ const uint b = (uint)(g * 4);+ const float4 a = acc[g];+ const uint c0 = col_base + b;+ if (c0 + 0 < (uint)n_no_padding) {+ dst[(c0 + 0) * (uint)m + row] = a.s0;+ }+ if (c0 + 1 < (uint)n_no_padding) {+ dst[(c0 + 1) * (uint)m + row] = a.s1;+ }+ if (c0 + 2 < (uint)n_no_padding) {+ dst[(c0 + 2) * (uint)m + row] = a.s2;+ }+ if (c0 + 3 < (uint)n_no_padding) {+ dst[(c0 + 3) * (uint)m + row] = a.s3;+ }+ }+#undef NGROUPS+}diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.clindex a3b39b6aa9..69cabb946c 100644--- a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q4_k_q8_1_dp4a.cl@@ -279,3 +279,141 @@ kernel void kernel_gemm_noshuffle_q4_k_q8_1_dp4a_wimg(}#undef NGROUPS}++// Stage each 32-K activation block as two uint4 values.+inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) {+ int r = 0;+ r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r);+ return r;+}++__attribute__((qcom_wave_pair_mode(1)))+kernel void kernel_gemm_noshuffle_q4_k_q8_1_dp4a_act_vload4(+ __global const ushort * src0_q,+ __global const uchar * src0_s,+ __global const half * src0_d,+ __global const half * src0_dm,+ __global const uint * src1_qa,+ __global const half * src1_da,+ __global const half * src1_sa,+ __global float * dst,+ ulong offsetd,+ int m,+ int n_no_padding,+ int k,+ uchar mask_d6,+ uchar mask_d4,+ uchar mask_hi2+) {+ dst = (global float *)((global char *)dst + offsetd);++ const uint lid = get_local_id(0);+ const uint block_id_m = get_global_id(1);+ const uint block_id_n = get_global_id(2);++ const uint row = block_id_m * 64 + lid;+ const uint col_base = block_id_n * TILESIZE_N;+ const bool row_valid = row < (uint)m;+ const uint rrow = row_valid ? row : 0;++ const uint k_u = (uint)k >> 2;+ const uint k_b = (uint)k >> 5;++ __local uint4 sh_qa4[TILESIZE_N][2];+ __local half sh_d[TILESIZE_N];+ __local half sh_s[TILESIZE_N];++#define NGROUPS (TILESIZE_N / 4)+ float4 acc[NGROUPS];+ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) {+ acc[g] = (float4)(0.0f);+ }++ for (uint step = 0; step < (uint)k; step += 32) {+ const uint sub = step >> 5;+ const uint sb_idx = step / QK_K;+ const uint sub_idx = sub & 7;++ const float dd = (float)src0_d [rrow + sb_idx * m];+ const float dmm = (float)src0_dm[rrow + sb_idx * m];+ global const uchar * sc = src0_s + sb_idx * K_SCALE_SIZE * (uint)m + rrow;+ uchar sv, mn;+ get_scale_min_k4(sub_idx, sc, (uint)m, &sv, &mn, mask_d6, mask_d4, mask_hi2);+ const float scale = dd * (float)sv;+ const float minv = dmm * (float)mn;++ const uint wbase = rrow + (step >> 2) * (uint)m;+ uint8 qw;+ qw.s0 = EXP4(src0_q[wbase + 0 * m]);+ qw.s1 = EXP4(src0_q[wbase + 1 * m]);+ qw.s2 = EXP4(src0_q[wbase + 2 * m]);+ qw.s3 = EXP4(src0_q[wbase + 3 * m]);+ qw.s4 = EXP4(src0_q[wbase + 4 * m]);+ qw.s5 = EXP4(src0_q[wbase + 5 * m]);+ qw.s6 = EXP4(src0_q[wbase + 6 * m]);+ qw.s7 = EXP4(src0_q[wbase + 7 * m]);++ for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) {+ const uint t = idx >> 1;+ const uint v = idx & 1;+ const uint c = col_base + t;+ sh_qa4[t][v] = (c < (uint)n_no_padding)+ ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2))+ : (uint4)(0u);+ }+ if (lid < TILESIZE_N) {+ const uint c = col_base + lid;+ sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;+ sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0;+ }+ barrier(CLK_LOCAL_MEM_FENCE);++#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))+ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) {+ const int b = g * 4;+ float4 rf;+#define DOTV(T) dot8_q8a_v(qw, sh_qa4[T][0], sh_qa4[T][1])+ rf.s0 = (float)DOTV(b+0);+ rf.s1 = (float)DOTV(b+1);+ rf.s2 = (float)DOTV(b+2);+ rf.s3 = (float)DOTV(b+3);+#undef DOTV+ acc[g] += scale * LD4(sh_d, b) * rf - minv * LD4(sh_s, b);+ }+#undef LD4+ barrier(CLK_LOCAL_MEM_FENCE);+ }++ if (!row_valid) {+ return;+ }++ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) {+ const uint b = (uint)(g * 4);+ const float4 a = acc[g];+ const uint c0 = col_base + b;+ if (c0 + 0 < (uint)n_no_padding) {+ dst[(c0 + 0) * (uint)m + row] = a.s0;+ }+ if (c0 + 1 < (uint)n_no_padding) {+ dst[(c0 + 1) * (uint)m + row] = a.s1;+ }+ if (c0 + 2 < (uint)n_no_padding) {+ dst[(c0 + 2) * (uint)m + row] = a.s2;+ }+ if (c0 + 3 < (uint)n_no_padding) {+ dst[(c0 + 3) * (uint)m + row] = a.s3;+ }+ }+#undef NGROUPS+}diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.clindex aaeed68f61..eef474b47f 100644--- a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_k_q8_1_dp4a.cl@@ -162,3 +162,141 @@ kernel void kernel_gemm_noshuffle_q5_k_q8_1_dp4a(}#undef NGROUPS}++// Stage each 32-K activation block as two uint4 values.+inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) {+ int r = 0;+ r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r);+ return r;+}++__attribute__((qcom_wave_pair_mode(1)))+kernel void kernel_gemm_noshuffle_q5_k_q8_1_dp4a_act_vload4(+ __global const ushort * src0_q,+ __global const uchar * src0_qh,+ __global const uchar * src0_s,+ __global const half * src0_d,+ __global const half * src0_dm,+ __global const uint * src1_qa,+ __global const half * src1_da,+ __global const half * src1_sa,+ __global float * dst,+ ulong offsetd,+ int m,+ int n_no_padding,+ int k,+ uchar mask_d6,+ uchar mask_d4,+ uchar mask_hi2+) {+ dst = (global float *)((global char *)dst + offsetd);++ const uint lid = get_local_id(0);+ const uint block_id_m = get_global_id(1);+ const uint block_id_n = get_global_id(2);++ const uint row = block_id_m * 64 + lid;+ const uint col_base = block_id_n * TILESIZE_N;+ const bool row_valid = row < (uint)m;+ const uint rrow = row_valid ? row : 0;++ const uint num_superblocks = (uint)k / QK_K;+ const uint k_u = (uint)k >> 2;+ const uint k_b = (uint)k >> 5;++ __local uint4 sh_qa4[TILESIZE_N][2];+ __local half sh_d[TILESIZE_N];+ __local half sh_s[TILESIZE_N];++#define NGROUPS (TILESIZE_N / 4)+ float4 acc[NGROUPS];+ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) {+ acc[g] = (float4)(0.0f);+ }++ for (uint step = 0; step < (uint)k; step += 32) {+ const uint sub = step >> 5;+ const uint sb_idx = step / QK_K;+ const uint sub_idx = sub & 7;++ const float dd = (float)src0_d [rrow + sb_idx * m];+ const float dmm = (float)src0_dm[rrow + sb_idx * m];+ global const uchar * sc = src0_s + rrow * num_superblocks * K_SCALE_SIZE + sb_idx * K_SCALE_SIZE;+ uchar sv, mn;+ get_scale_min_k4(sub_idx, sc, &sv, &mn, mask_d6, mask_d4, mask_hi2);+ const float scale = dd * (float)sv;+ const float minv = dmm * (float)mn;++ const uint wbase = rrow + (step >> 2) * (uint)m;+ const uint qhbase = rrow + (step >> 3) * (uint)m;+ uint8 qw;+#define QWU(u) ( EXP4((uint)src0_q[wbase + (uint)(u) * m]) \+ | EXP1( (uint)((src0_qh[qhbase + (uint)((u) >> 1) * m] >> (((u) & 1) * 4)) & 0x0Fu) ) )+ qw.s0 = QWU(0); qw.s1 = QWU(1); qw.s2 = QWU(2); qw.s3 = QWU(3);+ qw.s4 = QWU(4); qw.s5 = QWU(5); qw.s6 = QWU(6); qw.s7 = QWU(7);+#undef QWU++ for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) {+ const uint t = idx >> 1;+ const uint v = idx & 1;+ const uint c = col_base + t;+ sh_qa4[t][v] = (c < (uint)n_no_padding)+ ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2))+ : (uint4)(0u);+ }+ if (lid < TILESIZE_N) {+ const uint c = col_base + lid;+ sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;+ sh_s[lid] = (c < (uint)n_no_padding) ? src1_sa[c * k_b + sub] : (half)0;+ }+ barrier(CLK_LOCAL_MEM_FENCE);++#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))+ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) {+ const int b = g * 4;+ float4 rf;+#define DOTV(T) dot8_q8a_v(qw, sh_qa4[T][0], sh_qa4[T][1])+ rf.s0 = (float)DOTV(b+0);+ rf.s1 = (float)DOTV(b+1);+ rf.s2 = (float)DOTV(b+2);+ rf.s3 = (float)DOTV(b+3);+#undef DOTV+ acc[g] += scale * LD4(sh_d, b) * rf - minv * LD4(sh_s, b);+ }+#undef LD4+ barrier(CLK_LOCAL_MEM_FENCE);+ }++ if (!row_valid) {+ return;+ }++ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) {+ const uint b = (uint)(g * 4);+ const float4 a = acc[g];+ const uint c0 = col_base + b;+ if (c0 + 0 < (uint)n_no_padding) {+ dst[(c0 + 0) * (uint)m + row] = a.s0;+ }+ if (c0 + 1 < (uint)n_no_padding) {+ dst[(c0 + 1) * (uint)m + row] = a.s1;+ }+ if (c0 + 2 < (uint)n_no_padding) {+ dst[(c0 + 2) * (uint)m + row] = a.s2;+ }+ if (c0 + 3 < (uint)n_no_padding) {+ dst[(c0 + 3) * (uint)m + row] = a.s3;+ }+ }+#undef NGROUPS+}diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.clindex 382d79fdda..f17403a722 100644--- a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q6_k_q8_1_dp4a.cl@@ -142,3 +142,117 @@ kernel void kernel_gemm_noshuffle_q6_k_q8_1_dp4a(}#undef NGROUPS}++// Stage each 32-K activation block as two uint4 values.+__attribute__((qcom_wave_pair_mode(1)))+kernel void kernel_gemm_noshuffle_q6_k_q8_1_dp4a_act_vload4(+ __global const ushort * src0_ql,+ __global const uchar * src0_qh,+ __global const ushort * src0_s,+ __global const half * src0_d,+ __global const uint * src1_qa,+ __global const half * src1_da,+ __global float * dst,+ ulong offsetd,+ int m,+ int n_no_padding,+ int k+) {+ dst = (global float *)((global char *)dst + offsetd);++ const uint lid = get_local_id(0);+ const uint block_id_m = get_global_id(1);+ const uint block_id_n = get_global_id(2);++ const uint row = block_id_m * 64 + lid;+ const uint col_base = block_id_n * TILESIZE_N;+ const bool row_valid = row < (uint)m;+ const uint rrow = row_valid ? row : 0;++ const uint k_u = (uint)k >> 2;+ const uint k_b = (uint)k >> 5;++ __local uint4 sh_qa4[TILESIZE_N][2];+ __local half sh_d[TILESIZE_N];++#define NGROUPS (TILESIZE_N / 4)+ float4 acc[NGROUPS];+ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) {+ acc[g] = (float4)(0.0f);+ }++ for (uint step = 0; step < (uint)k; step += 32) {+ const uint sub = step >> 5;+ const uint sb_idx = step / QK_K;++ const float dd = (float)src0_d[rrow + sb_idx * m];+ const char2 sc = as_char2(src0_s[rrow + sub * m]);+ const float scale0 = dd * (float)sc.s0;+ const float scale1 = dd * (float)sc.s1;++ const uint wbase = rrow + (step >> 2) * (uint)m;+ uint qw[8];+ #pragma unroll+ for (int u = 0; u < 8; ++u) {+ const uint o = wbase + (uint)u * (uint)m;+ qw[u] = SIGN6(EXP4((uint)src0_ql[o]) | EXP2((uint)src0_qh[o]));+ }++ for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) {+ const uint t = idx >> 1;+ const uint v = idx & 1;+ const uint c = col_base + t;+ sh_qa4[t][v] = (c < (uint)n_no_padding)+ ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2))+ : (uint4)(0u);+ }+ if (lid < TILESIZE_N) {+ const uint c = col_base + lid;+ sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;+ }+ barrier(CLK_LOCAL_MEM_FENCE);++ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) {+ const int b = g * 4;+ float4 rf;+ #define DOT_TOK(j) { \+ const uint4 a0 = sh_qa4[b + (j)][0]; \+ const uint4 a1 = sh_qa4[b + (j)][1]; \+ const int raw1 = dot4_q8a(qw[0], qw[1], qw[2], qw[3], a0.x, a0.y, a0.z, a0.w); \+ const int raw2 = dot4_q8a(qw[4], qw[5], qw[6], qw[7], a1.x, a1.y, a1.z, a1.w); \+ rf.s##j = scale0 * (float)raw1 + scale1 * (float)raw2; \+ }+ DOT_TOK(0); DOT_TOK(1); DOT_TOK(2); DOT_TOK(3);+ #undef DOT_TOK+ const float4 ad = (float4)((float)sh_d[b+0], (float)sh_d[b+1], (float)sh_d[b+2], (float)sh_d[b+3]);+ acc[g] += ad * rf;+ }+ barrier(CLK_LOCAL_MEM_FENCE);+ }++ if (!row_valid) {+ return;+ }++ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) {+ const uint b = (uint)(g * 4);+ const float4 a = acc[g];+ const uint c0 = col_base + b;+ if (c0 + 0 < (uint)n_no_padding) {+ dst[(c0 + 0) * (uint)m + row] = a.s0;+ }+ if (c0 + 1 < (uint)n_no_padding) {+ dst[(c0 + 1) * (uint)m + row] = a.s1;+ }+ if (c0 + 2 < (uint)n_no_padding) {+ dst[(c0 + 2) * (uint)m + row] = a.s2;+ }+ if (c0 + 3 < (uint)n_no_padding) {+ dst[(c0 + 3) * (uint)m + row] = a.s3;+ }+ }+#undef NGROUPS+}diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.clindex a481636c23..32ff7b4f20 100644--- a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q8_0_q8_1_dp4a.cl@@ -210,3 +210,125 @@ kernel void kernel_gemm_noshuffle_q8_0_q8_1_dp4a_wimg(}#undef NGROUPS}++// Stage each 32-K activation block as two uint4 values.+inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) {+ int r = 0;+ r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r);+ r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r);+ return r;+}++__attribute__((qcom_wave_pair_mode(1)))+kernel void kernel_gemm_noshuffle_q8_0_q8_1_dp4a_act_vload4(+ __global const uint * src0_q,+ __global const half * src0_d,+ __global const uint * src1_qa,+ __global const half * src1_da,+ __global float * dst,+ ulong offsetd,+ int m,+ int n_no_padding,+ int k+) {+ dst = (global float *)((global char *)dst + offsetd);++ const uint lid = get_local_id(0);+ const uint block_id_m = get_global_id(1);+ const uint block_id_n = get_global_id(2);++ const uint row = block_id_m * 64 + lid;+ const uint col_base = block_id_n * TILESIZE_N;+ const bool row_valid = row < (uint)m;+ const uint rrow = row_valid ? row : 0;++ const uint k_u = (uint)k >> 2;+ const uint k_b = (uint)k >> 5;++ __local uint4 sh_qa4[TILESIZE_N][2];+ __local half sh_d[TILESIZE_N];++#define NGROUPS (TILESIZE_N / 4)+ float4 acc[NGROUPS];+ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) {+ acc[g] = (float4)(0.0f);+ }++ for (uint step = 0; step < (uint)k; step += 32) {+ const uint sub = step >> 5;++ const float d_w = (float)src0_d[rrow + sub * (uint)m];++ const uint wbase = rrow + (step >> 2) * (uint)m;+ uint8 qw;+ qw.s0 = src0_q[wbase + 0 * m];+ qw.s1 = src0_q[wbase + 1 * m];+ qw.s2 = src0_q[wbase + 2 * m];+ qw.s3 = src0_q[wbase + 3 * m];+ qw.s4 = src0_q[wbase + 4 * m];+ qw.s5 = src0_q[wbase + 5 * m];+ qw.s6 = src0_q[wbase + 6 * m];+ qw.s7 = src0_q[wbase + 7 * m];++ for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) {+ const uint t = idx >> 1;+ const uint v = idx & 1;+ const uint c = col_base + t;+ sh_qa4[t][v] = (c < (uint)n_no_padding)+ ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2))+ : (uint4)(0u);+ }+ if (lid < TILESIZE_N) {+ const uint c = col_base + lid;+ sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0;+ }+ barrier(CLK_LOCAL_MEM_FENCE);++#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3]))+ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) {+ const int b = g * 4;+ float4 rf;+#define DOTV(T) dot8_q8a_v(qw, sh_qa4[T][0], sh_qa4[T][1])+ rf.s0 = (float)DOTV(b+0);+ rf.s1 = (float)DOTV(b+1);+ rf.s2 = (float)DOTV(b+2);+ rf.s3 = (float)DOTV(b+3);+#undef DOTV+ acc[g] += d_w * LD4(sh_d, b) * rf;+ }+#undef LD4+ barrier(CLK_LOCAL_MEM_FENCE);+ }++ if (!row_valid) {+ return;+ }++ #pragma unroll+ for (int g = 0; g < NGROUPS; ++g) {+ const uint b = (uint)(g * 4);+ const float4 a = acc[g];+ const uint c0 = col_base + b;+ if (c0 + 0 < (uint)n_no_padding) {+ dst[(c0 + 0) * (uint)m + row] = a.s0;+ }+ if (c0 + 1 < (uint)n_no_padding) {+ dst[(c0 + 1) * (uint)m + row] = a.s1;+ }+ if (c0 + 2 < (uint)n_no_padding) {+ dst[(c0 + 2) * (uint)m + row] = a.s2;+ }+ if (c0 + 3 < (uint)n_no_padding) {+ dst[(c0 + 3) * (uint)m + row] = a.s3;+ }+ }+#undef NGROUPS+}