LLM inference in C/C++
C++ ggml
opencl: add q5_0/q5_1 gemm and gemv kernels for Adreno (#24319)
shaofeiqiba1df050f3dc
diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txtindex cd15d57323..82ce61d72c 100644--- a/ggml/src/ggml-opencl/CMakeLists.txt+++ b/ggml/src/ggml-opencl/CMakeLists.txt@@ -142,6 +142,10 @@ set(GGML_OPENCL_KERNELSgemm_noshuffle_q4_0_f32gemv_noshuffle_q4_1_f32gemm_noshuffle_q4_1_f32+ gemv_noshuffle_q5_0_f32+ gemm_noshuffle_q5_0_f32+ gemv_noshuffle_q5_1_f32+ gemm_noshuffle_q5_1_f32gemv_noshuffle_iq4_nl_f32gemm_noshuffle_iq4_nl_f32gemv_noshuffle_q8_0_f32diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cppindex d30579b945..ca2002424d 100644--- a/ggml/src/ggml-opencl/ggml-opencl.cpp+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp@@ -593,6 +593,10 @@ struct ggml_backend_opencl_context {cl_kernel kernel_restore_block_q4_0_noshuffle;cl_kernel kernel_convert_block_q4_1_noshuffle;cl_kernel kernel_restore_block_q4_1_noshuffle;+ cl_kernel kernel_convert_block_q5_0_noshuffle;+ cl_kernel kernel_restore_block_q5_0_noshuffle;+ cl_kernel kernel_convert_block_q5_1_noshuffle;+ cl_kernel kernel_restore_block_q5_1_noshuffle;cl_kernel kernel_convert_block_q4_K_noshuffle;cl_kernel kernel_restore_block_q4_K_noshuffle;cl_kernel kernel_convert_block_q4_K, kernel_restore_block_q4_K;@@ -829,6 +833,10 @@ struct ggml_backend_opencl_context {cl_kernel kernel_gemm_noshuffle_q6_K_f32;cl_kernel kernel_gemv_noshuffle_q5_k_f32;cl_kernel kernel_gemm_noshuffle_q5_k_f32;+ cl_kernel kernel_gemv_noshuffle_q5_0_f32;+ cl_kernel kernel_gemm_noshuffle_q5_0_f32;+ cl_kernel kernel_gemv_noshuffle_q5_1_f32;+ cl_kernel kernel_gemm_noshuffle_q5_1_f32;cl_kernel kernel_gemv_noshuffle_iq4_nl_f32;cl_kernel kernel_gemm_noshuffle_iq4_nl_f32;#endif // GGML_OPENCL_USE_ADRENO_KERNELS@@ -1152,6 +1160,10 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {CL_CHECK((backend_ctx->kernel_restore_block_q4_1_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q4_1_trans4_ns", &err), err));CL_CHECK((backend_ctx->kernel_convert_block_q5_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q5_0", &err), err));CL_CHECK((backend_ctx->kernel_restore_block_q5_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q5_0", &err), err));+ CL_CHECK((backend_ctx->kernel_convert_block_q5_0_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q5_0_noshuffle", &err), err));+ CL_CHECK((backend_ctx->kernel_restore_block_q5_0_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q5_0_noshuffle", &err), err));+ CL_CHECK((backend_ctx->kernel_convert_block_q5_1_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q5_1_noshuffle", &err), err));+ CL_CHECK((backend_ctx->kernel_restore_block_q5_1_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q5_1_noshuffle", &err), err));CL_CHECK((backend_ctx->kernel_convert_block_q5_0_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q5_0_trans4_ns", &err), err));CL_CHECK((backend_ctx->kernel_restore_block_q5_0_trans4_ns = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q5_0_trans4_ns", &err), err));CL_CHECK((backend_ctx->kernel_convert_block_q5_1 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q5_1", &err), err));@@ -3065,6 +3077,80 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {GGML_LOG_CONT(".");}+ // gemm_noshuffle_q5_0_f32+ {+#ifdef GGML_OPENCL_EMBED_KERNELS+ const std::string kernel_src {+ #include "gemm_noshuffle_q5_0_f32.cl.h"+ };+#else+ const std::string kernel_src = read_file("gemm_noshuffle_q5_0_f32.cl");+#endif+ cl_program prog = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_0_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_0_f32", &err), err));+ CL_CHECK(clReleaseProgram(prog));+ GGML_LOG_CONT(".");+ }++ // gemv_noshuffle_q5_0_f32+ {+ std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std ++ " -cl-mad-enable ";+ if (backend_ctx->has_vector_subgroup_broadcast) {+ CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST ";+ }++#ifdef GGML_OPENCL_EMBED_KERNELS+ const std::string kernel_src {+ #include "gemv_noshuffle_q5_0_f32.cl.h"+ };+#else+ const std::string kernel_src = read_file("gemv_noshuffle_q5_0_f32.cl");+#endif+ cl_program prog = build_program_from_source(+ backend_ctx->context, backend_ctx->device, kernel_src.c_str(), CL_gemv_compile_opts);+ CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q5_0_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q5_0_f32", &err), err));+ CL_CHECK(clReleaseProgram(prog));+ GGML_LOG_CONT(".");+ }++ // gemm_noshuffle_q5_1_f32+ {+#ifdef GGML_OPENCL_EMBED_KERNELS+ const std::string kernel_src {+ #include "gemm_noshuffle_q5_1_f32.cl.h"+ };+#else+ const std::string kernel_src = read_file("gemm_noshuffle_q5_1_f32.cl");+#endif+ cl_program prog = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_1_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q5_1_f32", &err), err));+ CL_CHECK(clReleaseProgram(prog));+ GGML_LOG_CONT(".");+ }++ // gemv_noshuffle_q5_1_f32+ {+ std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std ++ " -cl-mad-enable ";+ if (backend_ctx->has_vector_subgroup_broadcast) {+ CL_gemv_compile_opts += " -DVECTOR_SUB_GROUP_BROADCAST ";+ }++#ifdef GGML_OPENCL_EMBED_KERNELS+ const std::string kernel_src {+ #include "gemv_noshuffle_q5_1_f32.cl.h"+ };+#else+ const std::string kernel_src = read_file("gemv_noshuffle_q5_1_f32.cl");+#endif+ cl_program prog = build_program_from_source(+ backend_ctx->context, backend_ctx->device, kernel_src.c_str(), CL_gemv_compile_opts);+ CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q5_1_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q5_1_f32", &err), err));+ CL_CHECK(clReleaseProgram(prog));+ GGML_LOG_CONT(".");+ }+// gemm_noshuffle_iq4_nl_f32{#ifdef GGML_OPENCL_EMBED_KERNELS@@ -6107,15 +6193,16 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,return;}#endif // GGML_OPENCL_USE_ADRENO_KERNELS- cl_kernel kernel = backend_ctx->kernel_convert_block_q5_0;- cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type);++#ifdef GGML_OPENCL_USE_ADRENO_KERNELS+ if (use_adreno_kernels(backend_ctx, tensor)) {+ cl_kernel kernel = backend_ctx->kernel_convert_block_q5_0_noshuffle;CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device));CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs));CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh));CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d));- CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &n_blk));- size_t global_work_size[] = {(size_t)CEIL_DIV(n_blk, 64) * 64, 1, 1};+ size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1};size_t local_work_size[] = {64, 1, 1};cl_event evt;@@ -6124,7 +6211,39 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,CL_CHECK(clReleaseMemObject(data_device));tensor->extra = extra;++ int M = tensor->ne[1];+ int K = tensor->ne[0];+ GGML_ASSERT(K % 32 == 0);++ // Transpose qs as ushort+ transpose_2d_as_16b(backend_ctx, extra->qs, extra->qs, size_qs, K/4, M);+ // Transpose qh as uchar+ transpose_2d_as_8b(backend_ctx, extra->qh, extra->qh, size_qh, K/8, M);+ // Transpose d as ushort+ transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/32, M);+return;+ }+#endif // GGML_OPENCL_USE_ADRENO_KERNELS+ cl_kernel kernel = backend_ctx->kernel_convert_block_q5_0;+ cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type);+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device));+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs));+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh));+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d));+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &n_blk));++ size_t global_work_size[] = {(size_t)CEIL_DIV(n_blk, 64) * 64, 1, 1};+ size_t local_work_size[] = {64, 1, 1};++ cl_event evt;+ CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt));+ CL_CHECK(clWaitForEvents(1, &evt));+ CL_CHECK(clReleaseMemObject(data_device));++ tensor->extra = extra;+ return;}if (tensor->type == GGML_TYPE_Q5_1) {ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra;@@ -6225,6 +6344,42 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,return;}#endif // GGML_OPENCL_USE_ADRENO_KERNELS++#ifdef GGML_OPENCL_USE_ADRENO_KERNELS+ if (use_adreno_kernels(backend_ctx, tensor)) {+ cl_kernel kernel = backend_ctx->kernel_convert_block_q5_1_noshuffle;+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device));+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs));+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->qh));+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->d));+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->m));++ size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1};+ size_t local_work_size[] = {64, 1, 1};++ cl_event evt;+ CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt));+ CL_CHECK(clWaitForEvents(1, &evt));+ CL_CHECK(clReleaseMemObject(data_device));++ tensor->extra = extra;++ int M = tensor->ne[1];+ int K = tensor->ne[0];+ GGML_ASSERT(K % 32 == 0);++ // Transpose qs as ushort+ transpose_2d_as_16b(backend_ctx, extra->qs, extra->qs, size_qs, K/4, M);+ // Transpose qh as uchar+ transpose_2d_as_8b(backend_ctx, extra->qh, extra->qh, size_qh, K/8, M);+ // Transpose d as ushort+ transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/32, M);+ // Transpose m as ushort+ transpose_2d_as_16b(backend_ctx, extra->m, extra->m, size_m, K/32, M);++ return;+ }+#endif // GGML_OPENCL_USE_ADRENO_KERNELScl_kernel kernel = backend_ctx->kernel_convert_block_q5_1;cl_ulong n_blk = ggml_nelements(tensor)/ggml_blck_size(tensor->type);CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device));@@ -7299,6 +7454,48 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer,CL_CHECK(clReleaseMemObject(data_device));return;}+ if (use_adreno_kernels(backend_ctx, tensor)) {+ ggml_cl_buffer buf_trans_qs;+ ggml_cl_buffer buf_trans_qh;+ ggml_cl_buffer buf_trans_d;+ ggml_cl_buffer buf_unpacked;++ cl_int M = tensor->ne[1];+ cl_int K = tensor->ne[0];++ GGML_ASSERT(K % 32 == 0);++ size_t size_qs = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*ggml_blck_size(tensor->type)/2;+ size_t size_qh = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(int32_t);+ size_t size_d = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(ggml_fp16_t);++ buf_trans_qs.allocate(backend_ctx->context, size_qs);+ buf_trans_qh.allocate(backend_ctx->context, size_qh);+ buf_trans_d.allocate(backend_ctx->context, size_d);+ buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor));++ transpose_2d_as_16b(backend_ctx, extra->qs, buf_trans_qs.buffer, size_qs, M, K/4);+ transpose_2d_as_8b(backend_ctx, extra->qh, buf_trans_qh.buffer, size_qh, M, K/8);+ transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/32);++ cl_uchar mask_0F = 0x0F;+ cl_uchar mask_F0 = 0xF0;++ size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1};+ size_t local_work_size[] = {1, 1, 1};++ cl_kernel kernel = backend_ctx->kernel_restore_block_q5_0_noshuffle;+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_qs.buffer));+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_qh.buffer));+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_trans_d.buffer));+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_unpacked.buffer));+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_uchar), &mask_0F));+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask_F0));++ CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, NULL));+ CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL));+ return;+ }#endif // GGML_OPENCL_USE_ADRENO_KERNELScl_int err;@@ -7362,6 +7559,54 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer,CL_CHECK(clReleaseMemObject(data_device));return;}++ if (use_adreno_kernels(backend_ctx, tensor)) {+ ggml_cl_buffer buf_trans_qs;+ ggml_cl_buffer buf_trans_qh;+ ggml_cl_buffer buf_trans_d;+ ggml_cl_buffer buf_trans_m;+ ggml_cl_buffer buf_unpacked;++ cl_int M = tensor->ne[1];+ cl_int K = tensor->ne[0];+ GGML_ASSERT(K % 32 == 0);++ size_t size_qs = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*ggml_blck_size(tensor->type)/2;+ size_t size_qh = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(int32_t);+ size_t size_d = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(ggml_fp16_t);+ size_t size_m = (ggml_nelements(tensor)/ggml_blck_size(tensor->type))*sizeof(ggml_fp16_t);++ buf_trans_qs.allocate(backend_ctx->context, size_qs);+ buf_trans_qh.allocate(backend_ctx->context, size_qh);+ buf_trans_d.allocate(backend_ctx->context, size_d);+ buf_trans_m.allocate(backend_ctx->context, size_m);+ buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor));++ // Transpose back: from col-major to row-major+ transpose_2d_as_16b(backend_ctx, extra->qs, buf_trans_qs.buffer, size_qs, M, K/4);+ transpose_2d_as_8b(backend_ctx, extra->qh, buf_trans_qh.buffer, size_qh, M, K/8);+ transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/32);+ transpose_2d_as_16b(backend_ctx, extra->m, buf_trans_m.buffer, size_m, M, K/32);++ cl_uchar mask_0F = 0x0F;+ cl_uchar mask_F0 = 0xF0;++ size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1};+ size_t local_work_size[] = {1, 1, 1};++ cl_kernel kernel = backend_ctx->kernel_restore_block_q5_1_noshuffle;+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_qs.buffer));+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_qh.buffer));+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_trans_d.buffer));+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &buf_trans_m.buffer));+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &buf_unpacked.buffer));+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_uchar), &mask_0F));+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_uchar), &mask_F0));++ CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, NULL));+ CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL));+ return;+ }#endif // GGML_OPENCL_USE_ADRENO_KERNELScl_int err;cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE,@@ -12205,7 +12450,7 @@ static void ggml_cl_mul_mat_q4_1_f32_adreno(ggml_backend_t backend, const ggml_t#endif}-static void ggml_cl_mul_mat_iq4_nl_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+static void ggml_cl_mul_mat_q5_0_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {#ifdef GGML_OPENCL_USE_ADRENO_KERNELSGGML_ASSERT(src0);GGML_ASSERT(src0->extra);@@ -12218,17 +12463,17 @@ static void ggml_cl_mul_mat_iq4_nl_f32_adreno(ggml_backend_t backend, const ggmlggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;- ggml_tensor_extra_cl_iq4_nl * extra0_iq4_nl = (ggml_tensor_extra_cl_iq4_nl *)src0->extra;+ ggml_tensor_extra_cl_q5_0 * extra0_q5_0 = (ggml_tensor_extra_cl_q5_0 *)src0->extra;cl_ulong offset1 = extra1->offset + src1->view_offs;cl_ulong offsetd = extrad->offset + dst->view_offs;- const int ne00 = src0->ne[0];- const int ne01 = src0->ne[1];+ const int ne00 = src0->ne[0];+ const int ne01 = src0->ne[1];- const int ne1 = dst->ne[1];+ const int ne1 = dst->ne[1];- GGML_ASSERT(ne00 % 32 == 0);+ GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0);cl_context context = backend_ctx->context;cl_kernel kernel;@@ -12243,17 +12488,17 @@ static void ggml_cl_mul_mat_iq4_nl_f32_adreno(ggml_backend_t backend, const ggmlint K = ne00;if (ne1 == 1) {- cl_mem q_img = nullptr;+ cl_mem qs_img = nullptr;cl_mem b_sub_buf = nullptr;cl_mem b_img = nullptr;- // image for q- img_fmt = { CL_R, CL_UNSIGNED_INT32};+ // image for qs+ img_fmt = { CL_R, CL_UNSIGNED_INT32 };memset(&img_desc, 0, sizeof(img_desc));img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;img_desc.image_width = M * K / 2 / 4;- img_desc.buffer = extra0_iq4_nl->q;- CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));+ img_desc.buffer = extra0_q5_0->qs;+ CL_CHECK((qs_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));// subbuffer for activationsregion.origin = offset1;@@ -12268,22 +12513,23 @@ static void ggml_cl_mul_mat_iq4_nl_f32_adreno(ggml_backend_t backend, const ggmlimg_desc.buffer = b_sub_buf;CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));- kernel = backend_ctx->kernel_gemv_noshuffle_iq4_nl_f32;+ kernel = backend_ctx->kernel_gemv_noshuffle_q5_0_f32;- CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img));- CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq4_nl->d));- CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img));- CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device));- CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offsetd));- CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &ne00));- CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne01));+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qs_img));+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_0->qh));+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_0->d));+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &b_img));+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne00));+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne01));size_t local_work_size[3] = {64, 4, 1};size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, 4, 1};backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);- CL_CHECK(clReleaseMemObject(q_img));+ CL_CHECK(clReleaseMemObject(qs_img));CL_CHECK(clReleaseMemObject(b_sub_buf));CL_CHECK(clReleaseMemObject(b_img));} else {@@ -12291,6 +12537,7 @@ static void ggml_cl_mul_mat_iq4_nl_f32_adreno(ggml_backend_t backend, const ggmlcl_mem b_sub_buf_trans = nullptr;cl_mem b_img = nullptr;cl_mem b_img_trans = nullptr;+ cl_mem d_sub_buf = nullptr;// subbuffer for activationsregion.origin = offset1;@@ -12326,6 +12573,11 @@ static void ggml_cl_mul_mat_iq4_nl_f32_adreno(ggml_backend_t backend, const ggmlimg_desc.buffer = b_sub_buf_trans;CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err));+ // subbuffer for output+ region.origin = extrad->offset;+ region.size = M * N * sizeof(float);+ CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, CL_MEM_WRITE_ONLY, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));+// transpose activationsint height_B = N/4;if (height_B == 0) {@@ -12346,14 +12598,14 @@ static void ggml_cl_mul_mat_iq4_nl_f32_adreno(ggml_backend_t backend, const ggmlbackend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst);// gemm- kernel = backend_ctx->kernel_gemm_noshuffle_iq4_nl_f32;+ kernel = backend_ctx->kernel_gemm_noshuffle_q5_0_f32;int padded_N = N + padding;- CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq4_nl->q));- CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq4_nl->d));- CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img_trans));- CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device));- CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offsetd));+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_0->qs));+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_0->qh));+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_0->d));+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &b_img_trans));+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &d_sub_buf));CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &ne01));CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &padded_N));CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00));@@ -12368,6 +12620,7 @@ static void ggml_cl_mul_mat_iq4_nl_f32_adreno(ggml_backend_t backend, const ggmlCL_CHECK(clReleaseMemObject(b_sub_buf_trans));CL_CHECK(clReleaseMemObject(b_img));CL_CHECK(clReleaseMemObject(b_img_trans));+ CL_CHECK(clReleaseMemObject(d_sub_buf));}#elseGGML_UNUSED(backend);@@ -12377,7 +12630,7 @@ static void ggml_cl_mul_mat_iq4_nl_f32_adreno(ggml_backend_t backend, const ggml#endif}-static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+static void ggml_cl_mul_mat_q5_1_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {#ifdef GGML_OPENCL_USE_ADRENO_KERNELSGGML_ASSERT(src0);GGML_ASSERT(src0->extra);@@ -12386,34 +12639,21 @@ static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_tGGML_ASSERT(dst);GGML_ASSERT(dst->extra);- GGML_ASSERT(src0->type == GGML_TYPE_Q8_0);- GGML_ASSERT(src1->type == GGML_TYPE_F32);-ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;- ggml_tensor_extra_cl_q8_0 * extra0_q8_0 = (ggml_tensor_extra_cl_q8_0 *)src0->extra;+ ggml_tensor_extra_cl_q5_1 * extra0_q5_1 = (ggml_tensor_extra_cl_q5_1 *)src0->extra;cl_ulong offset1 = extra1->offset + src1->view_offs;cl_ulong offsetd = extrad->offset + dst->view_offs;- GGML_ASSERT(src1->view_offs == 0);- GGML_ASSERT(dst->view_offs == 0);-- const int ne00 = src0->ne[0];- const int ne01 = src0->ne[1];- const int ne02 = src0->ne[2];-- const int ne10 = src1->ne[0];- const int ne12 = src1->ne[2];+ const int ne00 = src0->ne[0];+ const int ne01 = src0->ne[1];- const int ne0 = dst->ne[0];- const int ne1 = dst->ne[1];+ const int ne1 = dst->ne[1];- GGML_ASSERT(ne00 == ne10);- GGML_ASSERT((ne00 % 32) == 0);- GGML_ASSERT(ne0 == ne01);+ GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0);cl_context context = backend_ctx->context;cl_kernel kernel;@@ -12428,17 +12668,384 @@ static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_tint K = ne00;if (ne1 == 1) {- cl_mem q_img = nullptr;+ cl_mem qs_img = nullptr;cl_mem b_sub_buf = nullptr;cl_mem b_img = nullptr;- // image for q- img_fmt = { CL_R, CL_UNSIGNED_INT32};+ // image for qs+ img_fmt = { CL_R, CL_UNSIGNED_INT32 };memset(&img_desc, 0, sizeof(img_desc));img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;- img_desc.image_width = M * K / 4;- img_desc.buffer = extra0_q8_0->q;- CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));+ img_desc.image_width = M * K / 2 / 4;+ img_desc.buffer = extra0_q5_1->qs;+ CL_CHECK((qs_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));++ // subbuffer for activations+ region.origin = offset1;+ region.size = K * N * sizeof(float);+ CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));++ // image for activations+ img_fmt = {CL_RGBA, CL_FLOAT};+ memset(&img_desc, 0, sizeof(img_desc));+ img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;+ img_desc.image_width = K * N / 4;+ img_desc.buffer = b_sub_buf;+ CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));++ kernel = backend_ctx->kernel_gemv_noshuffle_q5_1_f32;++ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qs_img));+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_1->qh));+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_1->d));+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_1->m));+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img));+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device));+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd));+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00));+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne01));++ size_t local_work_size[3] = {64, 4, 1};+ size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, 4, 1};++ backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);++ CL_CHECK(clReleaseMemObject(qs_img));+ CL_CHECK(clReleaseMemObject(b_sub_buf));+ CL_CHECK(clReleaseMemObject(b_img));+ } else {+ cl_mem b_sub_buf = nullptr;+ cl_mem b_sub_buf_trans = nullptr;+ cl_mem b_img = nullptr;+ cl_mem b_img_trans = nullptr;+ cl_mem d_sub_buf = nullptr;++ // subbuffer for activations+ region.origin = offset1;+ region.size = K * N * sizeof(float);+ CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));++ // image for activations+ img_fmt = {CL_RGBA, CL_FLOAT};+ memset(&img_desc, 0, sizeof(img_desc));+ img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;+ img_desc.image_width = K * N / 4;+ img_desc.buffer = b_sub_buf;+ CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));++ // pad N to multiple of 8+ int extra_elements = N % 8;+ int padding = 0;+ if (extra_elements > 0){+ padding = 8 - extra_elements;+ }++ // subbuffer for transposed activations+ region.origin = 0;+ region.size = K * (N + padding) * sizeof(float)/2;+ backend_ctx->prealloc_act_trans.allocate(context, region.size);+ CL_CHECK((b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));++ // image for transposed activations+ img_fmt = {CL_RGBA, CL_HALF_FLOAT};+ memset(&img_desc, 0, sizeof(img_desc));+ img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;+ img_desc.image_width = K * (N + padding) / 4;+ img_desc.buffer = b_sub_buf_trans;+ CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err));++ // subbuffer for output+ region.origin = extrad->offset;+ region.size = M * N * sizeof(float);+ CL_CHECK((d_sub_buf = clCreateSubBuffer(extrad->data_device, CL_MEM_WRITE_ONLY, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));++ // transpose activations+ int height_B = N/4;+ if (height_B == 0) {+ height_B = 1;+ }+ int width_B = K/4;+ int padded_height_B = (N + padding)/4;++ kernel = backend_ctx->kernel_transpose_32_16;+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img));+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans));+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B));+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B));+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B));++ size_t local_work_size_t[2] = { 1, 16 };+ size_t global_work_size_t[2] = { (size_t)width_B, (size_t)padded_height_B };+ backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst);++ // gemm+ kernel = backend_ctx->kernel_gemm_noshuffle_q5_1_f32;+ int padded_N = N + padding;++ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q5_1->qs));+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q5_1->qh));+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_q5_1->d));+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_q5_1->m));+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img_trans));+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &d_sub_buf));+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne01));+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &padded_N));+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne00));+ CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_int), &ne1));++ size_t global_work_size[3] = {(size_t)CEIL_DIV(ne1, 8), (size_t)CEIL_DIV(ne01, 4), 1};+ size_t local_work_size[3] = {1, 128, 1};++ backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);++ CL_CHECK(clReleaseMemObject(b_sub_buf));+ CL_CHECK(clReleaseMemObject(b_sub_buf_trans));+ CL_CHECK(clReleaseMemObject(b_img));+ CL_CHECK(clReleaseMemObject(b_img_trans));+ CL_CHECK(clReleaseMemObject(d_sub_buf));+ }+#else+ GGML_UNUSED(backend);+ GGML_UNUSED(src0);+ GGML_UNUSED(src1);+ GGML_UNUSED(dst);+#endif+}++static void ggml_cl_mul_mat_iq4_nl_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+#ifdef GGML_OPENCL_USE_ADRENO_KERNELS+ GGML_ASSERT(src0);+ GGML_ASSERT(src0->extra);+ GGML_ASSERT(src1);+ GGML_ASSERT(src1->extra);+ GGML_ASSERT(dst);+ GGML_ASSERT(dst->extra);++ ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;++ ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;+ ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;+ ggml_tensor_extra_cl_iq4_nl * extra0_iq4_nl = (ggml_tensor_extra_cl_iq4_nl *)src0->extra;++ cl_ulong offset1 = extra1->offset + src1->view_offs;+ cl_ulong offsetd = extrad->offset + dst->view_offs;++ const int ne00 = src0->ne[0];+ const int ne01 = src0->ne[1];++ const int ne1 = dst->ne[1];++ GGML_ASSERT(ne00 % 32 == 0);++ cl_context context = backend_ctx->context;+ cl_kernel kernel;++ cl_int err;+ cl_image_format img_fmt;+ cl_image_desc img_desc;+ cl_buffer_region region;++ int M = ne01;+ int N = ne1;+ int K = ne00;++ if (ne1 == 1) {+ cl_mem q_img = nullptr;+ cl_mem b_sub_buf = nullptr;+ cl_mem b_img = nullptr;++ // image for q+ img_fmt = { CL_R, CL_UNSIGNED_INT32};+ memset(&img_desc, 0, sizeof(img_desc));+ img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;+ img_desc.image_width = M * K / 2 / 4;+ img_desc.buffer = extra0_iq4_nl->q;+ CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));++ // subbuffer for activations+ region.origin = offset1;+ region.size = K * N * sizeof(float);+ CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));++ // image for activations+ img_fmt = {CL_RGBA, CL_FLOAT};+ memset(&img_desc, 0, sizeof(img_desc));+ img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;+ img_desc.image_width = K * N / 4;+ img_desc.buffer = b_sub_buf;+ CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));++ kernel = backend_ctx->kernel_gemv_noshuffle_iq4_nl_f32;++ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img));+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq4_nl->d));+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img));+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device));+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offsetd));+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &ne00));+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &ne01));++ size_t local_work_size[3] = {64, 4, 1};+ size_t global_work_size[3] = {(size_t)CEIL_DIV(ne01/2, 64)*64, 4, 1};++ backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);++ CL_CHECK(clReleaseMemObject(q_img));+ CL_CHECK(clReleaseMemObject(b_sub_buf));+ CL_CHECK(clReleaseMemObject(b_img));+ } else {+ cl_mem b_sub_buf = nullptr;+ cl_mem b_sub_buf_trans = nullptr;+ cl_mem b_img = nullptr;+ cl_mem b_img_trans = nullptr;++ // subbuffer for activations+ region.origin = offset1;+ region.size = K * N * sizeof(float);+ CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));++ // image for activations+ img_fmt = {CL_RGBA, CL_FLOAT};+ memset(&img_desc, 0, sizeof(img_desc));+ img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;+ img_desc.image_width = K * N / 4;+ img_desc.buffer = b_sub_buf;+ CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));++ // pad N to multiple of 8+ int extra_elements = N % 8;+ int padding = 0;+ if (extra_elements > 0){+ padding = 8 - extra_elements;+ }++ // subbuffer for transposed activations+ region.origin = 0;+ region.size = K * (N + padding) * sizeof(float)/2;+ backend_ctx->prealloc_act_trans.allocate(context, region.size);+ CL_CHECK((b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));++ // image for transposed activations+ img_fmt = {CL_RGBA, CL_HALF_FLOAT};+ memset(&img_desc, 0, sizeof(img_desc));+ img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;+ img_desc.image_width = K * (N + padding) / 4;+ img_desc.buffer = b_sub_buf_trans;+ CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err));++ // transpose activations+ int height_B = N/4;+ if (height_B == 0) {+ height_B = 1;+ }+ int width_B = K/4;+ int padded_height_B = (N + padding)/4;++ kernel = backend_ctx->kernel_transpose_32_16;+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img));+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans));+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B));+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B));+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B));++ size_t local_work_size_t[2] = { 1, 16 };+ size_t global_work_size_t[2] = { (size_t)width_B, (size_t)padded_height_B };+ backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst);++ // gemm+ kernel = backend_ctx->kernel_gemm_noshuffle_iq4_nl_f32;+ int padded_N = N + padding;++ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq4_nl->q));+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq4_nl->d));+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img_trans));+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device));+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offsetd));+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_int), &ne01));+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_int), &padded_N));+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_int), &ne00));+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_int), &ne1));++ size_t global_work_size[3] = {(size_t)CEIL_DIV(ne1, 8), (size_t)CEIL_DIV(ne01, 4), 1};+ size_t local_work_size[3] = {1, 128, 1};++ backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);++ CL_CHECK(clReleaseMemObject(b_sub_buf));+ CL_CHECK(clReleaseMemObject(b_sub_buf_trans));+ CL_CHECK(clReleaseMemObject(b_img));+ CL_CHECK(clReleaseMemObject(b_img_trans));+ }+#else+ GGML_UNUSED(backend);+ GGML_UNUSED(src0);+ GGML_UNUSED(src1);+ GGML_UNUSED(dst);+#endif+}++static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+#ifdef GGML_OPENCL_USE_ADRENO_KERNELS+ GGML_ASSERT(src0);+ GGML_ASSERT(src0->extra);+ GGML_ASSERT(src1);+ GGML_ASSERT(src1->extra);+ GGML_ASSERT(dst);+ GGML_ASSERT(dst->extra);++ GGML_ASSERT(src0->type == GGML_TYPE_Q8_0);+ GGML_ASSERT(src1->type == GGML_TYPE_F32);++ ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;++ ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;+ ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;+ ggml_tensor_extra_cl_q8_0 * extra0_q8_0 = (ggml_tensor_extra_cl_q8_0 *)src0->extra;++ cl_ulong offset1 = extra1->offset + src1->view_offs;+ cl_ulong offsetd = extrad->offset + dst->view_offs;++ GGML_ASSERT(src1->view_offs == 0);+ GGML_ASSERT(dst->view_offs == 0);++ const int ne00 = src0->ne[0];+ const int ne01 = src0->ne[1];+ const int ne02 = src0->ne[2];++ const int ne10 = src1->ne[0];+ const int ne12 = src1->ne[2];++ const int ne0 = dst->ne[0];+ const int ne1 = dst->ne[1];++ GGML_ASSERT(ne00 == ne10);+ GGML_ASSERT((ne00 % 32) == 0);+ GGML_ASSERT(ne0 == ne01);++ cl_context context = backend_ctx->context;+ cl_kernel kernel;++ cl_int err;+ cl_image_format img_fmt;+ cl_image_desc img_desc;+ cl_buffer_region region;++ int M = ne01;+ int N = ne1;+ int K = ne00;++ if (ne1 == 1) {+ cl_mem q_img = nullptr;+ cl_mem b_sub_buf = nullptr;+ cl_mem b_img = nullptr;++ // image for q+ img_fmt = { CL_R, CL_UNSIGNED_INT32};+ memset(&img_desc, 0, sizeof(img_desc));+ img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;+ img_desc.image_width = M * K / 4;+ img_desc.buffer = extra0_q8_0->q;+ CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));// create a sub_buffer for Bregion.origin = offset1;@@ -13243,6 +13850,18 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, coreturn;}+ // q5_0 x fp32+ if (src0t == GGML_TYPE_Q5_0 && src1t == GGML_TYPE_F32) {+ ggml_cl_mul_mat_q5_0_f32_adreno(backend, src0, src1, dst);+ return;+ }++ // q5_1 x fp32+ if (src0t == GGML_TYPE_Q5_1 && src1t == GGML_TYPE_F32) {+ ggml_cl_mul_mat_q5_1_f32_adreno(backend, src0, src1, dst);+ return;+ }+// iq4_nl x fp32if (src0t == GGML_TYPE_IQ4_NL && src1t == GGML_TYPE_F32) {ggml_cl_mul_mat_iq4_nl_f32_adreno(backend, src0, src1, dst);diff --git a/ggml/src/ggml-opencl/kernels/cvt.cl b/ggml/src/ggml-opencl/kernels/cvt.clindex d07f0a1a02..226b127ab3 100644--- a/ggml/src/ggml-opencl/kernels/cvt.cl+++ b/ggml/src/ggml-opencl/kernels/cvt.cl@@ -584,6 +584,60 @@ kernel void kernel_restore_block_q5_0(}}+kernel void kernel_convert_block_q5_0_noshuffle(+ global struct block_q5_0 * src0,+ global uchar * dst_q,+ global uint * dst_qh,+ global half * dst_d+) {+ global struct block_q5_0 * b = (global struct block_q5_0 *) src0 + get_global_id(0);+ global uchar * q = (global uchar *) dst_q + QK5_0/2*get_global_id(0);+ global uint * qh = (global uint *) dst_qh + get_global_id(0);+ global half * d = (global half *) dst_d + get_global_id(0);++ *d = b->d;+ *qh = *((global uint *)(b->qh));++ for (int i = 0; i < QK5_0/4; ++i) {+ uchar x0 = b->qs[2*i + 0];+ uchar x1 = b->qs[2*i + 1];++ q[i + 0 ] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4);+ q[i + QK5_0/4] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0);++#ifdef ADRENO_GPU+ if (get_global_id(0) == 65536*4096) {+ printf("%04x - %02x\n", *(global ushort*)d, ((x0 & 0xF0) >> 4) | (x1 & 0xF0));+ }+#endif+ }+}++kernel void kernel_restore_block_q5_0_noshuffle(+ global uchar * src_q,+ global uint * src_qh,+ global half * src_d,+ global struct block_q5_0 * dst,+ uchar mask_0F,+ uchar mask_F0+) {+ global struct block_q5_0 * b = (global struct block_q5_0 *) dst + get_global_id(0);+ global uchar * q = (global uchar *) src_q + QK5_0/2*get_global_id(0);+ global uint * qh = (global uint *) src_qh + get_global_id(0);+ global half * d = (global half *) src_d + get_global_id(0);++ b->d = *d;+ *((global uint *)(b->qh)) = *qh;++ for (int i = 0; i < QK5_0/4; ++i) {+ uchar x0 = q[i + 0 ];+ uchar x1 = q[i + QK5_0/4];++ b->qs[2*i + 0] = convert_uchar((x0 & mask_0F) | ((x1 & mask_0F) << 4));+ b->qs[2*i + 1] = convert_uchar(((x0 & mask_F0) >> 4) | (x1 & mask_F0));+ }+}+kernel void kernel_convert_block_q5_0_trans4_ns(__global struct block_q5_0 * src0,__global uint * dst_qs,@@ -736,6 +790,66 @@ kernel void kernel_restore_block_q5_1(}}+kernel void kernel_convert_block_q5_1_noshuffle(+ global struct block_q5_1 * src0,+ global uchar * dst_q,+ global uint * dst_qh,+ global half * dst_d,+ global half * dst_m+) {+ global struct block_q5_1 * b = (global struct block_q5_1 *) src0 + get_global_id(0);+ global uchar * q = (global uchar *) dst_q + QK5_1/2*get_global_id(0);+ global uint * qh = (global uint *) dst_qh + get_global_id(0);+ global half * d = (global half *) dst_d + get_global_id(0);+ global half * m = (global half *) dst_m + get_global_id(0);++ *d = b->d;+ *m = b->m;+ *qh = *((global uint *)(b->qh));++ for (int i = 0; i < QK5_1/4; ++i) {+ uchar x0 = b->qs[2*i + 0];+ uchar x1 = b->qs[2*i + 1];++ q[i + 0 ] = convert_uchar(x0 & 0x0F) | convert_uchar((x1 & 0x0F) << 4);+ q[i + QK5_1/4] = convert_uchar((x0 & 0xF0) >> 4) | convert_uchar(x1 & 0xF0);++#ifdef ADRENO_GPU+ if (get_global_id(0) == 65536*4096) {+ printf("%04x - %02x\n", *(global ushort*)d, ((x0 & 0xF0) >> 4) | (x1 & 0xF0));+ }+#endif+ }+}++kernel void kernel_restore_block_q5_1_noshuffle(+ global uchar * src_q,+ global uint * src_qh,+ global half * src_d,+ global half * src_m,+ global struct block_q5_1 * dst,+ uchar mask_0F,+ uchar mask_F0+) {+ global struct block_q5_1 * b = (global struct block_q5_1 *) dst + get_global_id(0);+ global uchar * q = (global uchar *) src_q + QK5_1/2*get_global_id(0);+ global uint * qh = (global uint *) src_qh + get_global_id(0);+ global half * d = (global half *) src_d + get_global_id(0);+ global half * m = (global half *) src_m + get_global_id(0);++ b->d = *d;+ b->m = *m;+ *((global uint *)(b->qh)) = *qh;++ for (int i = 0; i < QK5_1/4; ++i) {+ uchar x0 = q[i + 0 ];+ uchar x1 = q[i + QK5_1/4];++ b->qs[2*i + 0] = convert_uchar((x0 & mask_0F) | ((x1 & mask_0F) << 4));+ b->qs[2*i + 1] = convert_uchar(((x0 & mask_F0) >> 4) | (x1 & mask_F0));+ }+}+kernel void kernel_convert_block_q5_1_trans4_ns(__global struct block_q5_1 * src0,__global uint * dst_qs,diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_f32.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_f32.clnew file mode 100644index 0000000000..1d6bd48005--- /dev/null+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_0_f32.cl@@ -0,0 +1,131 @@+#pragma OPENCL EXTENSION cl_khr_fp16 : enable+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable++#ifdef cl_qcom_reqd_sub_group_size+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable+#define ADRENO_GPU 1+#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))+#endif++#ifdef ADRENO_GPU+REQD_SUBGROUP_SIZE_128+#endif++kernel void kernel_gemm_noshuffle_q5_0_f32(+ global const ushort * src0_qs, // quantized A+ global const uchar * src0_qh, // 5th bits+ global const half * src0_d, // A scales+ __read_only image1d_buffer_t src1, // B (1d image)+ global float * dst, // C+ int m, // M+ int n, // N with padding+ int k, // K+ int n_no_padding // N without padding+) {++ int n_4 = n >> 2;++ int gy = get_global_id(0);+ int gx = get_global_id(1);+ int gx_2 = gx << 2;++ half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0;+ half8 B;+ half4 dequantized_weights;++ global const ushort * weight_ptr = src0_qs + gx_2;+ global const uchar * qh_ptr = src0_qh + gx_2;+ global const half * scale_ptr = src0_d + gx_2;++ for (int i = 0; i < k; i += 4) {++ B.s0123 = read_imageh(src1, gy*2 + i*n_4);+ B.s4567 = read_imageh(src1, gy*2 + i*n_4 + 1);++ ushort4 bits4 = vload4(0, weight_ptr + (i >> 2)*m);+ uchar4 bits1 = vload4(0, qh_ptr + (i >> 3)*m);+ uchar4 qh = bits1 >> (uchar4)(i & 4);++ half4 scale = vload4(0, scale_ptr + (i >> 5)*m);++ // j=0+ dequantized_weights.s0 = (convert_half((bits4.s0 & 0x000F) | ((qh.s0 & 0x01) << 4)) - 16.0h) * scale.s0;+ dequantized_weights.s1 = (convert_half((bits4.s1 & 0x000F) | ((qh.s1 & 0x01) << 4)) - 16.0h) * scale.s1;+ dequantized_weights.s2 = (convert_half((bits4.s2 & 0x000F) | ((qh.s2 & 0x01) << 4)) - 16.0h) * scale.s2;+ dequantized_weights.s3 = (convert_half((bits4.s3 & 0x000F) | ((qh.s3 & 0x01) << 4)) - 16.0h) * scale.s3;+ c0 += B * dequantized_weights.s0;+ c1 += B * dequantized_weights.s1;+ c2 += B * dequantized_weights.s2;+ c3 += B * dequantized_weights.s3;++ // j=1+ B.s0123 = read_imageh(src1, gy*2 + (i+1)*n_4);+ B.s4567 = read_imageh(src1, gy*2 + (i+1)*n_4 + 1);+ dequantized_weights.s0 = (convert_half(((bits4.s0 & 0x00F0) >> 4) | ((qh.s0 & 0x02) << 3)) - 16.0h) * scale.s0;+ dequantized_weights.s1 = (convert_half(((bits4.s1 & 0x00F0) >> 4) | ((qh.s1 & 0x02) << 3)) - 16.0h) * scale.s1;+ dequantized_weights.s2 = (convert_half(((bits4.s2 & 0x00F0) >> 4) | ((qh.s2 & 0x02) << 3)) - 16.0h) * scale.s2;+ dequantized_weights.s3 = (convert_half(((bits4.s3 & 0x00F0) >> 4) | ((qh.s3 & 0x02) << 3)) - 16.0h) * scale.s3;+ c0 += B * dequantized_weights.s0;+ c1 += B * dequantized_weights.s1;+ c2 += B * dequantized_weights.s2;+ c3 += B * dequantized_weights.s3;++ // j=2+ B.s0123 = read_imageh(src1, gy*2 + (i+2)*n_4);+ B.s4567 = read_imageh(src1, gy*2 + (i+2)*n_4 + 1);+ dequantized_weights.s0 = (convert_half(((bits4.s0 & 0x0F00) >> 8) | ((qh.s0 & 0x04) << 2)) - 16.0h) * scale.s0;+ dequantized_weights.s1 = (convert_half(((bits4.s1 & 0x0F00) >> 8) | ((qh.s1 & 0x04) << 2)) - 16.0h) * scale.s1;+ dequantized_weights.s2 = (convert_half(((bits4.s2 & 0x0F00) >> 8) | ((qh.s2 & 0x04) << 2)) - 16.0h) * scale.s2;+ dequantized_weights.s3 = (convert_half(((bits4.s3 & 0x0F00) >> 8) | ((qh.s3 & 0x04) << 2)) - 16.0h) * scale.s3;+ c0 += B * dequantized_weights.s0;+ c1 += B * dequantized_weights.s1;+ c2 += B * dequantized_weights.s2;+ c3 += B * dequantized_weights.s3;++ // j=3+ B.s0123 = read_imageh(src1, gy*2 + (i+3)*n_4);+ B.s4567 = read_imageh(src1, gy*2 + (i+3)*n_4 + 1);+ dequantized_weights.s0 = (convert_half(((bits4.s0 & 0xF000) >> 12) | ((qh.s0 & 0x08) << 1)) - 16.0h) * scale.s0;+ dequantized_weights.s1 = (convert_half(((bits4.s1 & 0xF000) >> 12) | ((qh.s1 & 0x08) << 1)) - 16.0h) * scale.s1;+ dequantized_weights.s2 = (convert_half(((bits4.s2 & 0xF000) >> 12) | ((qh.s2 & 0x08) << 1)) - 16.0h) * scale.s2;+ dequantized_weights.s3 = (convert_half(((bits4.s3 & 0xF000) >> 12) | ((qh.s3 & 0x08) << 1)) - 16.0h) * scale.s3;+ c0 += B * dequantized_weights.s0;+ c1 += B * dequantized_weights.s1;+ c2 += B * dequantized_weights.s2;+ c3 += B * dequantized_weights.s3;+ }++ int idx = (gy<<3)*m + (gx<<2);++ if(idx+3 < m*n_no_padding){+ vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx);+ idx += m;+ }+ if(idx+3 < m*n_no_padding){+ vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx);+ idx += m;+ }+ if(idx+3 < m*n_no_padding){+ vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx);+ idx += m;+ }+ if(idx+3 < m*n_no_padding){+ vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx);+ idx += m;+ }+ if(idx+3 < m*n_no_padding){+ vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx);+ idx += m;+ }+ if(idx+3 < m*n_no_padding){+ vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx);+ idx += m;+ }+ if(idx+3 < m*n_no_padding){+ vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx);+ idx += m;+ }+ if(idx+3 < m*n_no_padding){+ vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx);+ }+}diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_1_f32.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_1_f32.clnew file mode 100644index 0000000000..94b4ef6cac--- /dev/null+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q5_1_f32.cl@@ -0,0 +1,134 @@+#pragma OPENCL EXTENSION cl_khr_fp16 : enable+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable++#ifdef cl_qcom_reqd_sub_group_size+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable+#define ADRENO_GPU 1+#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))+#endif++#ifdef ADRENO_GPU+REQD_SUBGROUP_SIZE_128+#endif++kernel void kernel_gemm_noshuffle_q5_1_f32(+ global const ushort * src0_qs, // quantized A+ global const uchar * src0_qh, // 5th bits+ global const half * src0_d, // A scales+ global const half * src0_m, // A mins+ __read_only image1d_buffer_t src1, // B (1d image)+ global float * dst, // C+ int m, // M+ int n, // N with padding+ int k, // K+ int n_no_padding // N without padding+) {++ int n_4 = n >> 2;++ int gy = get_global_id(0);+ int gx = get_global_id(1);+ int gx_2 = gx << 2;++ half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0;+ half8 B;+ half4 dequantized_weights;++ global const ushort * weight_ptr = src0_qs + gx_2;+ global const uchar * qh_ptr = src0_qh + gx_2;+ global const half * scale_ptr = src0_d + gx_2;+ global const half * min_ptr = src0_m + gx_2;++ for (int i = 0; i < k; i += 4) {++ B.s0123 = read_imageh(src1, gy*2 + i*n_4);+ B.s4567 = read_imageh(src1, gy*2 + i*n_4 + 1);++ ushort4 bits4 = vload4(0, weight_ptr + (i >> 2)*m);+ uchar4 bits1 = vload4(0, qh_ptr + (i >> 3)*m);+ uchar4 qh = bits1 >> (uchar4)(i & 4);++ half4 scale = vload4(0, scale_ptr + (i >> 5)*m);+ half4 minv = vload4(0, min_ptr + (i >> 5)*m);++ // j=0+ dequantized_weights.s0 = convert_half((bits4.s0 & 0x000F) | ((qh.s0 & 0x01) << 4)) * scale.s0 + minv.s0;+ dequantized_weights.s1 = convert_half((bits4.s1 & 0x000F) | ((qh.s1 & 0x01) << 4)) * scale.s1 + minv.s1;+ dequantized_weights.s2 = convert_half((bits4.s2 & 0x000F) | ((qh.s2 & 0x01) << 4)) * scale.s2 + minv.s2;+ dequantized_weights.s3 = convert_half((bits4.s3 & 0x000F) | ((qh.s3 & 0x01) << 4)) * scale.s3 + minv.s3;+ c0 += B * dequantized_weights.s0;+ c1 += B * dequantized_weights.s1;+ c2 += B * dequantized_weights.s2;+ c3 += B * dequantized_weights.s3;++ // j=1+ B.s0123 = read_imageh(src1, gy*2 + (i+1)*n_4);+ B.s4567 = read_imageh(src1, gy*2 + (i+1)*n_4 + 1);+ dequantized_weights.s0 = convert_half(((bits4.s0 & 0x00F0) >> 4) | ((qh.s0 & 0x02) << 3)) * scale.s0 + minv.s0;+ dequantized_weights.s1 = convert_half(((bits4.s1 & 0x00F0) >> 4) | ((qh.s1 & 0x02) << 3)) * scale.s1 + minv.s1;+ dequantized_weights.s2 = convert_half(((bits4.s2 & 0x00F0) >> 4) | ((qh.s2 & 0x02) << 3)) * scale.s2 + minv.s2;+ dequantized_weights.s3 = convert_half(((bits4.s3 & 0x00F0) >> 4) | ((qh.s3 & 0x02) << 3)) * scale.s3 + minv.s3;+ c0 += B * dequantized_weights.s0;+ c1 += B * dequantized_weights.s1;+ c2 += B * dequantized_weights.s2;+ c3 += B * dequantized_weights.s3;++ // j=2+ B.s0123 = read_imageh(src1, gy*2 + (i+2)*n_4);+ B.s4567 = read_imageh(src1, gy*2 + (i+2)*n_4 + 1);+ dequantized_weights.s0 = convert_half(((bits4.s0 & 0x0F00) >> 8) | ((qh.s0 & 0x04) << 2)) * scale.s0 + minv.s0;+ dequantized_weights.s1 = convert_half(((bits4.s1 & 0x0F00) >> 8) | ((qh.s1 & 0x04) << 2)) * scale.s1 + minv.s1;+ dequantized_weights.s2 = convert_half(((bits4.s2 & 0x0F00) >> 8) | ((qh.s2 & 0x04) << 2)) * scale.s2 + minv.s2;+ dequantized_weights.s3 = convert_half(((bits4.s3 & 0x0F00) >> 8) | ((qh.s3 & 0x04) << 2)) * scale.s3 + minv.s3;+ c0 += B * dequantized_weights.s0;+ c1 += B * dequantized_weights.s1;+ c2 += B * dequantized_weights.s2;+ c3 += B * dequantized_weights.s3;++ // j=3+ B.s0123 = read_imageh(src1, gy*2 + (i+3)*n_4);+ B.s4567 = read_imageh(src1, gy*2 + (i+3)*n_4 + 1);+ dequantized_weights.s0 = convert_half(((bits4.s0 & 0xF000) >> 12) | ((qh.s0 & 0x08) << 1)) * scale.s0 + minv.s0;+ dequantized_weights.s1 = convert_half(((bits4.s1 & 0xF000) >> 12) | ((qh.s1 & 0x08) << 1)) * scale.s1 + minv.s1;+ dequantized_weights.s2 = convert_half(((bits4.s2 & 0xF000) >> 12) | ((qh.s2 & 0x08) << 1)) * scale.s2 + minv.s2;+ dequantized_weights.s3 = convert_half(((bits4.s3 & 0xF000) >> 12) | ((qh.s3 & 0x08) << 1)) * scale.s3 + minv.s3;+ c0 += B * dequantized_weights.s0;+ c1 += B * dequantized_weights.s1;+ c2 += B * dequantized_weights.s2;+ c3 += B * dequantized_weights.s3;+ }++ int idx = (gy<<3)*m + (gx<<2);++ if(idx+3 < m*n_no_padding){+ vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx);+ idx += m;+ }+ if(idx+3 < m*n_no_padding){+ vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx);+ idx += m;+ }+ if(idx+3 < m*n_no_padding){+ vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx);+ idx += m;+ }+ if(idx+3 < m*n_no_padding){+ vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx);+ idx += m;+ }+ if(idx+3 < m*n_no_padding){+ vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx);+ idx += m;+ }+ if(idx+3 < m*n_no_padding){+ vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx);+ idx += m;+ }+ if(idx+3 < m*n_no_padding){+ vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx);+ idx += m;+ }+ if(idx+3 < m*n_no_padding){+ vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx);+ }+}diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.clnew file mode 100644index 0000000000..c228f717a9--- /dev/null+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_0_f32.cl@@ -0,0 +1,291 @@+#pragma OPENCL EXTENSION cl_khr_fp16 : enable+#pragma OPENCL EXTENSION cl_khr_subgroups : enable++#ifdef cl_qcom_reqd_sub_group_size+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable+#define ADRENO_GPU 1+#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))+#endif++#define QK5_0 32+#define NSUBGROUPS 4+#define SUBGROUP_SIZE 64++#define dequantizeBlockAccum_ns_q5_0_sgbroadcast_1_hi(total_sums, bits4, bits1, scale, y) \+ float shared_y; \+ shared_y = sub_group_broadcast(y.s0, 0); \+ total_sums.s0 += (((bits4.s0 & 0x000F) | (((bits1.s0 ) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \+ total_sums.s1 += (((bits4.s1 & 0x000F) | (((bits1.s4 ) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \+ shared_y = sub_group_broadcast(y.s1, 0); \+ total_sums.s0 += ((((bits4.s0 & 0x00F0) >> 4) | (((bits1.s0 >> 1) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \+ total_sums.s1 += ((((bits4.s1 & 0x00F0) >> 4) | (((bits1.s4 >> 1) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \+ shared_y = sub_group_broadcast(y.s2, 0); \+ total_sums.s0 += ((((bits4.s0 & 0x0F00) >> 8) | (((bits1.s0 >> 2) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \+ total_sums.s1 += ((((bits4.s1 & 0x0F00) >> 8) | (((bits1.s4 >> 2) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \+ shared_y = sub_group_broadcast(y.s3, 0); \+ total_sums.s0 += ((((bits4.s0 & 0xF000) >> 12) | (((bits1.s0 >> 3) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \+ total_sums.s1 += ((((bits4.s1 & 0xF000) >> 12) | (((bits1.s4 >> 3) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \+ shared_y = sub_group_broadcast(y.s4, 0); \+ total_sums.s0 += (((bits4.s2 & 0x000F) | (((bits1.s0 >> 4) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \+ total_sums.s1 += (((bits4.s3 & 0x000F) | (((bits1.s4 >> 4) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \+ shared_y = sub_group_broadcast(y.s5, 0); \+ total_sums.s0 += ((((bits4.s2 & 0x00F0) >> 4) | (((bits1.s0 >> 5) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \+ total_sums.s1 += ((((bits4.s3 & 0x00F0) >> 4) | (((bits1.s4 >> 5) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \+ shared_y = sub_group_broadcast(y.s6, 0); \+ total_sums.s0 += ((((bits4.s2 & 0x0F00) >> 8) | (((bits1.s0 >> 6) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \+ total_sums.s1 += ((((bits4.s3 & 0x0F00) >> 8) | (((bits1.s4 >> 6) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \+ shared_y = sub_group_broadcast(y.s7, 0); \+ total_sums.s0 += ((((bits4.s2 & 0xF000) >> 12) | (((bits1.s0 >> 7) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \+ total_sums.s1 += ((((bits4.s3 & 0xF000) >> 12) | (((bits1.s4 >> 7) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \+ shared_y = sub_group_broadcast(y.s0, 1); \+ total_sums.s0 += (((bits4.s4 & 0x000F) | (((bits1.s1 ) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \+ total_sums.s1 += (((bits4.s5 & 0x000F) | (((bits1.s5 ) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \+ shared_y = sub_group_broadcast(y.s1, 1); \+ total_sums.s0 += ((((bits4.s4 & 0x00F0) >> 4) | (((bits1.s1 >> 1) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \+ total_sums.s1 += ((((bits4.s5 & 0x00F0) >> 4) | (((bits1.s5 >> 1) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \+ shared_y = sub_group_broadcast(y.s2, 1); \+ total_sums.s0 += ((((bits4.s4 & 0x0F00) >> 8) | (((bits1.s1 >> 2) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \+ total_sums.s1 += ((((bits4.s5 & 0x0F00) >> 8) | (((bits1.s5 >> 2) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \+ shared_y = sub_group_broadcast(y.s3, 1); \+ total_sums.s0 += ((((bits4.s4 & 0xF000) >> 12) | (((bits1.s1 >> 3) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \+ total_sums.s1 += ((((bits4.s5 & 0xF000) >> 12) | (((bits1.s5 >> 3) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \+ shared_y = sub_group_broadcast(y.s4, 1); \+ total_sums.s0 += (((bits4.s6 & 0x000F) | (((bits1.s1 >> 4) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \+ total_sums.s1 += (((bits4.s7 & 0x000F) | (((bits1.s5 >> 4) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \+ shared_y = sub_group_broadcast(y.s5, 1); \+ total_sums.s0 += ((((bits4.s6 & 0x00F0) >> 4) | (((bits1.s1 >> 5) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \+ total_sums.s1 += ((((bits4.s7 & 0x00F0) >> 4) | (((bits1.s5 >> 5) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \+ shared_y = sub_group_broadcast(y.s6, 1); \+ total_sums.s0 += ((((bits4.s6 & 0x0F00) >> 8) | (((bits1.s1 >> 6) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \+ total_sums.s1 += ((((bits4.s7 & 0x0F00) >> 8) | (((bits1.s5 >> 6) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \+ shared_y = sub_group_broadcast(y.s7, 1); \+ total_sums.s0 += ((((bits4.s6 & 0xF000) >> 12) | (((bits1.s1 >> 7) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \+ total_sums.s1 += ((((bits4.s7 & 0xF000) >> 12) | (((bits1.s5 >> 7) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \+++#define dequantizeBlockAccum_ns_q5_0_sgbroadcast_1_lo(total_sums, bits4, bits1, scale, y) \+ shared_y = sub_group_broadcast(y.s0, 2); \+ total_sums.s0 += (((bits4.s0 & 0x000F) | (((bits1.s2 ) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \+ total_sums.s1 += (((bits4.s1 & 0x000F) | (((bits1.s6 ) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \+ shared_y = sub_group_broadcast(y.s1, 2); \+ total_sums.s0 += ((((bits4.s0 & 0x00F0) >> 4) | (((bits1.s2 >> 1) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \+ total_sums.s1 += ((((bits4.s1 & 0x00F0) >> 4) | (((bits1.s6 >> 1) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \+ shared_y = sub_group_broadcast(y.s2, 2); \+ total_sums.s0 += ((((bits4.s0 & 0x0F00) >> 8) | (((bits1.s2 >> 2) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \+ total_sums.s1 += ((((bits4.s1 & 0x0F00) >> 8) | (((bits1.s6 >> 2) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \+ shared_y = sub_group_broadcast(y.s3, 2); \+ total_sums.s0 += ((((bits4.s0 & 0xF000) >> 12) | (((bits1.s2 >> 3) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \+ total_sums.s1 += ((((bits4.s1 & 0xF000) >> 12) | (((bits1.s6 >> 3) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \+ shared_y = sub_group_broadcast(y.s4, 2); \+ total_sums.s0 += (((bits4.s2 & 0x000F) | (((bits1.s2 >> 4) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \+ total_sums.s1 += (((bits4.s3 & 0x000F) | (((bits1.s6 >> 4) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \+ shared_y = sub_group_broadcast(y.s5, 2); \+ total_sums.s0 += ((((bits4.s2 & 0x00F0) >> 4) | (((bits1.s2 >> 5) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \+ total_sums.s1 += ((((bits4.s3 & 0x00F0) >> 4) | (((bits1.s6 >> 5) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \+ shared_y = sub_group_broadcast(y.s6, 2); \+ total_sums.s0 += ((((bits4.s2 & 0x0F00) >> 8) | (((bits1.s2 >> 6) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \+ total_sums.s1 += ((((bits4.s3 & 0x0F00) >> 8) | (((bits1.s6 >> 6) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \+ shared_y = sub_group_broadcast(y.s7, 2); \+ total_sums.s0 += ((((bits4.s2 & 0xF000) >> 12) | (((bits1.s2 >> 7) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \+ total_sums.s1 += ((((bits4.s3 & 0xF000) >> 12) | (((bits1.s6 >> 7) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \+ shared_y = sub_group_broadcast(y.s0, 3); \+ total_sums.s0 += (((bits4.s4 & 0x000F) | (((bits1.s3 ) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \+ total_sums.s1 += (((bits4.s5 & 0x000F) | (((bits1.s7 ) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \+ shared_y = sub_group_broadcast(y.s1, 3); \+ total_sums.s0 += ((((bits4.s4 & 0x00F0) >> 4) | (((bits1.s3 >> 1) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \+ total_sums.s1 += ((((bits4.s5 & 0x00F0) >> 4) | (((bits1.s7 >> 1) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \+ shared_y = sub_group_broadcast(y.s2, 3); \+ total_sums.s0 += ((((bits4.s4 & 0x0F00) >> 8) | (((bits1.s3 >> 2) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \+ total_sums.s1 += ((((bits4.s5 & 0x0F00) >> 8) | (((bits1.s7 >> 2) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \+ shared_y = sub_group_broadcast(y.s3, 3); \+ total_sums.s0 += ((((bits4.s4 & 0xF000) >> 12) | (((bits1.s3 >> 3) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \+ total_sums.s1 += ((((bits4.s5 & 0xF000) >> 12) | (((bits1.s7 >> 3) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \+ shared_y = sub_group_broadcast(y.s4, 3); \+ total_sums.s0 += (((bits4.s6 & 0x000F) | (((bits1.s3 >> 4) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \+ total_sums.s1 += (((bits4.s7 & 0x000F) | (((bits1.s7 >> 4) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \+ shared_y = sub_group_broadcast(y.s5, 3); \+ total_sums.s0 += ((((bits4.s6 & 0x00F0) >> 4) | (((bits1.s3 >> 5) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \+ total_sums.s1 += ((((bits4.s7 & 0x00F0) >> 4) | (((bits1.s7 >> 5) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \+ shared_y = sub_group_broadcast(y.s6, 3); \+ total_sums.s0 += ((((bits4.s6 & 0x0F00) >> 8) | (((bits1.s3 >> 6) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \+ total_sums.s1 += ((((bits4.s7 & 0x0F00) >> 8) | (((bits1.s7 >> 6) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \+ shared_y = sub_group_broadcast(y.s7, 3); \+ total_sums.s0 += ((((bits4.s6 & 0xF000) >> 12) | (((bits1.s3 >> 7) & 0x01) << 4)) - 16) * scale.s0 * shared_y; \+ total_sums.s1 += ((((bits4.s7 & 0xF000) >> 12) | (((bits1.s7 >> 7) & 0x01) << 4)) - 16) * scale.s1 * shared_y; \+++#define dequantizeBlockAccum_ns_q5_0_sgbroadcast_8_hi(total_sums, bits4, bits1, scale, y) \+ float8 shared_y; \+ shared_y = sub_group_broadcast(y, 0); \+ total_sums.s0 += (((bits4.s0 & 0x000F) | (((bits1.s0 ) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s0; \+ total_sums.s0 += ((((bits4.s0 & 0x00F0) >> 4) | (((bits1.s0 >> 1) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s1; \+ total_sums.s0 += ((((bits4.s0 & 0x0F00) >> 8) | (((bits1.s0 >> 2) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s2; \+ total_sums.s0 += ((((bits4.s0 & 0xF000) >> 12) | (((bits1.s0 >> 3) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s3; \+ total_sums.s0 += (((bits4.s2 & 0x000F) | (((bits1.s0 >> 4) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s4; \+ total_sums.s0 += ((((bits4.s2 & 0x00F0) >> 4) | (((bits1.s0 >> 5) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s5; \+ total_sums.s0 += ((((bits4.s2 & 0x0F00) >> 8) | (((bits1.s0 >> 6) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s6; \+ total_sums.s0 += ((((bits4.s2 & 0xF000) >> 12) | (((bits1.s0 >> 7) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s7; \+ total_sums.s1 += (((bits4.s1 & 0x000F) | (((bits1.s4 ) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s0; \+ total_sums.s1 += ((((bits4.s1 & 0x00F0) >> 4) | (((bits1.s4 >> 1) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s1; \+ total_sums.s1 += ((((bits4.s1 & 0x0F00) >> 8) | (((bits1.s4 >> 2) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s2; \+ total_sums.s1 += ((((bits4.s1 & 0xF000) >> 12) | (((bits1.s4 >> 3) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s3; \+ total_sums.s1 += (((bits4.s3 & 0x000F) | (((bits1.s4 >> 4) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s4; \+ total_sums.s1 += ((((bits4.s3 & 0x00F0) >> 4) | (((bits1.s4 >> 5) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s5; \+ total_sums.s1 += ((((bits4.s3 & 0x0F00) >> 8) | (((bits1.s4 >> 6) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s6; \+ total_sums.s1 += ((((bits4.s3 & 0xF000) >> 12) | (((bits1.s4 >> 7) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s7; \+ shared_y = sub_group_broadcast(y, 1); \+ total_sums.s0 += (((bits4.s4 & 0x000F) | (((bits1.s1 ) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s0; \+ total_sums.s0 += ((((bits4.s4 & 0x00F0) >> 4) | (((bits1.s1 >> 1) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s1; \+ total_sums.s0 += ((((bits4.s4 & 0x0F00) >> 8) | (((bits1.s1 >> 2) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s2; \+ total_sums.s0 += ((((bits4.s4 & 0xF000) >> 12) | (((bits1.s1 >> 3) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s3; \+ total_sums.s0 += (((bits4.s6 & 0x000F) | (((bits1.s1 >> 4) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s4; \+ total_sums.s0 += ((((bits4.s6 & 0x00F0) >> 4) | (((bits1.s1 >> 5) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s5; \+ total_sums.s0 += ((((bits4.s6 & 0x0F00) >> 8) | (((bits1.s1 >> 6) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s6; \+ total_sums.s0 += ((((bits4.s6 & 0xF000) >> 12) | (((bits1.s1 >> 7) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s7; \+ total_sums.s1 += (((bits4.s5 & 0x000F) | (((bits1.s5 ) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s0; \+ total_sums.s1 += ((((bits4.s5 & 0x00F0) >> 4) | (((bits1.s5 >> 1) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s1; \+ total_sums.s1 += ((((bits4.s5 & 0x0F00) >> 8) | (((bits1.s5 >> 2) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s2; \+ total_sums.s1 += ((((bits4.s5 & 0xF000) >> 12) | (((bits1.s5 >> 3) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s3; \+ total_sums.s1 += (((bits4.s7 & 0x000F) | (((bits1.s5 >> 4) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s4; \+ total_sums.s1 += ((((bits4.s7 & 0x00F0) >> 4) | (((bits1.s5 >> 5) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s5; \+ total_sums.s1 += ((((bits4.s7 & 0x0F00) >> 8) | (((bits1.s5 >> 6) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s6; \+ total_sums.s1 += ((((bits4.s7 & 0xF000) >> 12) | (((bits1.s5 >> 7) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s7; \+++#define dequantizeBlockAccum_ns_q5_0_sgbroadcast_8_lo(total_sums, bits4, bits1, scale, y) \+ shared_y = sub_group_broadcast(y, 2); \+ total_sums.s0 += (((bits4.s0 & 0x000F) | (((bits1.s2 ) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s0; \+ total_sums.s0 += ((((bits4.s0 & 0x00F0) >> 4) | (((bits1.s2 >> 1) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s1; \+ total_sums.s0 += ((((bits4.s0 & 0x0F00) >> 8) | (((bits1.s2 >> 2) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s2; \+ total_sums.s0 += ((((bits4.s0 & 0xF000) >> 12) | (((bits1.s2 >> 3) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s3; \+ total_sums.s0 += (((bits4.s2 & 0x000F) | (((bits1.s2 >> 4) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s4; \+ total_sums.s0 += ((((bits4.s2 & 0x00F0) >> 4) | (((bits1.s2 >> 5) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s5; \+ total_sums.s0 += ((((bits4.s2 & 0x0F00) >> 8) | (((bits1.s2 >> 6) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s6; \+ total_sums.s0 += ((((bits4.s2 & 0xF000) >> 12) | (((bits1.s2 >> 7) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s7; \+ total_sums.s1 += (((bits4.s1 & 0x000F) | (((bits1.s6 ) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s0; \+ total_sums.s1 += ((((bits4.s1 & 0x00F0) >> 4) | (((bits1.s6 >> 1) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s1; \+ total_sums.s1 += ((((bits4.s1 & 0x0F00) >> 8) | (((bits1.s6 >> 2) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s2; \+ total_sums.s1 += ((((bits4.s1 & 0xF000) >> 12) | (((bits1.s6 >> 3) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s3; \+ total_sums.s1 += (((bits4.s3 & 0x000F) | (((bits1.s6 >> 4) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s4; \+ total_sums.s1 += ((((bits4.s3 & 0x00F0) >> 4) | (((bits1.s6 >> 5) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s5; \+ total_sums.s1 += ((((bits4.s3 & 0x0F00) >> 8) | (((bits1.s6 >> 6) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s6; \+ total_sums.s1 += ((((bits4.s3 & 0xF000) >> 12) | (((bits1.s6 >> 7) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s7; \+ shared_y = sub_group_broadcast(y, 3); \+ total_sums.s0 += (((bits4.s4 & 0x000F) | (((bits1.s3 ) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s0; \+ total_sums.s0 += ((((bits4.s4 & 0x00F0) >> 4) | (((bits1.s3 >> 1) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s1; \+ total_sums.s0 += ((((bits4.s4 & 0x0F00) >> 8) | (((bits1.s3 >> 2) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s2; \+ total_sums.s0 += ((((bits4.s4 & 0xF000) >> 12) | (((bits1.s3 >> 3) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s3; \+ total_sums.s0 += (((bits4.s6 & 0x000F) | (((bits1.s3 >> 4) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s4; \+ total_sums.s0 += ((((bits4.s6 & 0x00F0) >> 4) | (((bits1.s3 >> 5) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s5; \+ total_sums.s0 += ((((bits4.s6 & 0x0F00) >> 8) | (((bits1.s3 >> 6) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s6; \+ total_sums.s0 += ((((bits4.s6 & 0xF000) >> 12) | (((bits1.s3 >> 7) & 0x01) << 4)) - 16) * scale.s0 * shared_y.s7; \+ total_sums.s1 += (((bits4.s5 & 0x000F) | (((bits1.s7 ) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s0; \+ total_sums.s1 += ((((bits4.s5 & 0x00F0) >> 4) | (((bits1.s7 >> 1) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s1; \+ total_sums.s1 += ((((bits4.s5 & 0x0F00) >> 8) | (((bits1.s7 >> 2) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s2; \+ total_sums.s1 += ((((bits4.s5 & 0xF000) >> 12) | (((bits1.s7 >> 3) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s3; \+ total_sums.s1 += (((bits4.s7 & 0x000F) | (((bits1.s7 >> 4) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s4; \+ total_sums.s1 += ((((bits4.s7 & 0x00F0) >> 4) | (((bits1.s7 >> 5) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s5; \+ total_sums.s1 += ((((bits4.s7 & 0x0F00) >> 8) | (((bits1.s7 >> 6) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s6; \+ total_sums.s1 += ((((bits4.s7 & 0xF000) >> 12) | (((bits1.s7 >> 7) & 0x01) << 4)) - 16) * scale.s1 * shared_y.s7; \++#ifdef ADRENO_GPU+REQD_SUBGROUP_SIZE_64+#endif+__kernel void kernel_gemv_noshuffle_q5_0_f32(+ __read_only image1d_buffer_t src0_qs, // quantized A+ global ushort * src0_qh, // 5th bits+ global half2 * src0_d, // A scales+ __read_only image1d_buffer_t src1, // B activations+ global float * dst,+ ulong offsetd,+ int ne00, // K+ int ne01) // M+{+ uint groupId = get_local_id(1);+ uint gid = get_global_id(0);+ ushort slid = get_sub_group_local_id();++ uint K = ne00;+ uint M = ne01;++ uint LINE_STRIDE_A = M / 2;+ uint BLOCK_STRIDE_A = NSUBGROUPS * M;++ private uint4 regA;+ private half2 regS;+ private float8 regB;++ private float2 totalSum = (float2)(0.0f);++ for (uint k = groupId; k < (K / QK5_0); k += NSUBGROUPS) {+ regS = src0_d[gid + k * LINE_STRIDE_A];++ ushort4 qh_raw;+ qh_raw.s0 = src0_qh[gid + (4*k + 0) * LINE_STRIDE_A];+ qh_raw.s1 = src0_qh[gid + (4*k + 1) * LINE_STRIDE_A];+ qh_raw.s2 = src0_qh[gid + (4*k + 2) * LINE_STRIDE_A];+ qh_raw.s3 = src0_qh[gid + (4*k + 3) * LINE_STRIDE_A];++ uchar8 raw = as_uchar8(qh_raw);+ uchar8 qh_bytes = (uchar8)(raw.s0, raw.s2, raw.s4, raw.s6,+ raw.s1, raw.s3, raw.s5, raw.s7);++ // Load activations+ if (slid < 4) {+ regB.s0123 = read_imagef(src1, (slid * 2 + k * 8));+ regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8));+ }++ regA.s0 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;+ regA.s1 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;+ regA.s2 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;+ regA.s3 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;++#ifdef VECTOR_SUB_GROUP_BROADCAST+ dequantizeBlockAccum_ns_q5_0_sgbroadcast_8_hi(totalSum, as_ushort8(regA), qh_bytes, regS, regB);+#else+ dequantizeBlockAccum_ns_q5_0_sgbroadcast_1_hi(totalSum, as_ushort8(regA), qh_bytes, regS, regB);+#endif // VECTOR_SUB_GROUP_BROADCAST++ regA.s0 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;+ regA.s1 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;+ regA.s2 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;+ regA.s3 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;+#ifdef VECTOR_SUB_GROUP_BROADCAST+ dequantizeBlockAccum_ns_q5_0_sgbroadcast_8_lo(totalSum, as_ushort8(regA), qh_bytes, regS, regB);+#else+ dequantizeBlockAccum_ns_q5_0_sgbroadcast_1_lo(totalSum, as_ushort8(regA), qh_bytes, regS, regB);+#endif // VECTOR_SUB_GROUP_BROADCAST+ }++ // reduction in local memory, assumes #wave=4+ local float2 reduceLM[SUBGROUP_SIZE * 3];+ if (groupId == 1) {+ reduceLM[SUBGROUP_SIZE * 0 + slid] = totalSum;+ }+ if (groupId == 2) {+ reduceLM[SUBGROUP_SIZE * 1 + slid] = totalSum;+ }+ if (groupId == 3) {+ reduceLM[SUBGROUP_SIZE * 2 + slid] = totalSum;+ }++ barrier(CLK_LOCAL_MEM_FENCE);++ if (groupId == 0) {+ totalSum += reduceLM[SUBGROUP_SIZE * 0 + slid];+ }+ if (groupId == 0) {+ totalSum += reduceLM[SUBGROUP_SIZE * 1 + slid];+ }+ if (groupId == 0) {+ totalSum += reduceLM[SUBGROUP_SIZE * 2 + slid];+ }++ // 2 outputs per fiber in wave 0+ if (groupId == 0) {+ dst = (global float*)((global char*)dst + offsetd);+ vstore2(totalSum, 0, &(dst[gid * 2]));+ }++}diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.clnew file mode 100644index 0000000000..daf1308ea4--- /dev/null+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q5_1_f32.cl@@ -0,0 +1,294 @@+#pragma OPENCL EXTENSION cl_khr_fp16 : enable+#pragma OPENCL EXTENSION cl_khr_subgroups : enable++#ifdef cl_qcom_reqd_sub_group_size+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable+#define ADRENO_GPU 1+#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))+#endif++#define QK5_1 32+#define NSUBGROUPS 4+#define SUBGROUP_SIZE 64++#define dequantizeBlockAccum_ns_q5_1_sgbroadcast_1_hi(total_sums, bits4, bits1, scale, minv, y) \+ float shared_y; \+ shared_y = sub_group_broadcast(y.s0, 0); \+ total_sums.s0 += (((bits4.s0 & 0x000F) | (((bits1.s0 ) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \+ total_sums.s1 += (((bits4.s1 & 0x000F) | (((bits1.s4 ) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \+ shared_y = sub_group_broadcast(y.s1, 0); \+ total_sums.s0 += ((((bits4.s0 & 0x00F0) >> 4) | (((bits1.s0 >> 1) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \+ total_sums.s1 += ((((bits4.s1 & 0x00F0) >> 4) | (((bits1.s4 >> 1) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \+ shared_y = sub_group_broadcast(y.s2, 0); \+ total_sums.s0 += ((((bits4.s0 & 0x0F00) >> 8) | (((bits1.s0 >> 2) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \+ total_sums.s1 += ((((bits4.s1 & 0x0F00) >> 8) | (((bits1.s4 >> 2) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \+ shared_y = sub_group_broadcast(y.s3, 0); \+ total_sums.s0 += ((((bits4.s0 & 0xF000) >> 12) | (((bits1.s0 >> 3) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \+ total_sums.s1 += ((((bits4.s1 & 0xF000) >> 12) | (((bits1.s4 >> 3) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \+ shared_y = sub_group_broadcast(y.s4, 0); \+ total_sums.s0 += (((bits4.s2 & 0x000F) | (((bits1.s0 >> 4) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \+ total_sums.s1 += (((bits4.s3 & 0x000F) | (((bits1.s4 >> 4) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \+ shared_y = sub_group_broadcast(y.s5, 0); \+ total_sums.s0 += ((((bits4.s2 & 0x00F0) >> 4) | (((bits1.s0 >> 5) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \+ total_sums.s1 += ((((bits4.s3 & 0x00F0) >> 4) | (((bits1.s4 >> 5) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \+ shared_y = sub_group_broadcast(y.s6, 0); \+ total_sums.s0 += ((((bits4.s2 & 0x0F00) >> 8) | (((bits1.s0 >> 6) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \+ total_sums.s1 += ((((bits4.s3 & 0x0F00) >> 8) | (((bits1.s4 >> 6) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \+ shared_y = sub_group_broadcast(y.s7, 0); \+ total_sums.s0 += ((((bits4.s2 & 0xF000) >> 12) | (((bits1.s0 >> 7) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \+ total_sums.s1 += ((((bits4.s3 & 0xF000) >> 12) | (((bits1.s4 >> 7) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \+ shared_y = sub_group_broadcast(y.s0, 1); \+ total_sums.s0 += (((bits4.s4 & 0x000F) | (((bits1.s1 ) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \+ total_sums.s1 += (((bits4.s5 & 0x000F) | (((bits1.s5 ) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \+ shared_y = sub_group_broadcast(y.s1, 1); \+ total_sums.s0 += ((((bits4.s4 & 0x00F0) >> 4) | (((bits1.s1 >> 1) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \+ total_sums.s1 += ((((bits4.s5 & 0x00F0) >> 4) | (((bits1.s5 >> 1) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \+ shared_y = sub_group_broadcast(y.s2, 1); \+ total_sums.s0 += ((((bits4.s4 & 0x0F00) >> 8) | (((bits1.s1 >> 2) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \+ total_sums.s1 += ((((bits4.s5 & 0x0F00) >> 8) | (((bits1.s5 >> 2) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \+ shared_y = sub_group_broadcast(y.s3, 1); \+ total_sums.s0 += ((((bits4.s4 & 0xF000) >> 12) | (((bits1.s1 >> 3) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \+ total_sums.s1 += ((((bits4.s5 & 0xF000) >> 12) | (((bits1.s5 >> 3) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \+ shared_y = sub_group_broadcast(y.s4, 1); \+ total_sums.s0 += (((bits4.s6 & 0x000F) | (((bits1.s1 >> 4) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \+ total_sums.s1 += (((bits4.s7 & 0x000F) | (((bits1.s5 >> 4) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \+ shared_y = sub_group_broadcast(y.s5, 1); \+ total_sums.s0 += ((((bits4.s6 & 0x00F0) >> 4) | (((bits1.s1 >> 5) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \+ total_sums.s1 += ((((bits4.s7 & 0x00F0) >> 4) | (((bits1.s5 >> 5) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \+ shared_y = sub_group_broadcast(y.s6, 1); \+ total_sums.s0 += ((((bits4.s6 & 0x0F00) >> 8) | (((bits1.s1 >> 6) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \+ total_sums.s1 += ((((bits4.s7 & 0x0F00) >> 8) | (((bits1.s5 >> 6) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \+ shared_y = sub_group_broadcast(y.s7, 1); \+ total_sums.s0 += ((((bits4.s6 & 0xF000) >> 12) | (((bits1.s1 >> 7) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \+ total_sums.s1 += ((((bits4.s7 & 0xF000) >> 12) | (((bits1.s5 >> 7) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \+++#define dequantizeBlockAccum_ns_q5_1_sgbroadcast_1_lo(total_sums, bits4, bits1, scale, minv, y) \+ shared_y = sub_group_broadcast(y.s0, 2); \+ total_sums.s0 += (((bits4.s0 & 0x000F) | (((bits1.s2 ) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \+ total_sums.s1 += (((bits4.s1 & 0x000F) | (((bits1.s6 ) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \+ shared_y = sub_group_broadcast(y.s1, 2); \+ total_sums.s0 += ((((bits4.s0 & 0x00F0) >> 4) | (((bits1.s2 >> 1) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \+ total_sums.s1 += ((((bits4.s1 & 0x00F0) >> 4) | (((bits1.s6 >> 1) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \+ shared_y = sub_group_broadcast(y.s2, 2); \+ total_sums.s0 += ((((bits4.s0 & 0x0F00) >> 8) | (((bits1.s2 >> 2) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \+ total_sums.s1 += ((((bits4.s1 & 0x0F00) >> 8) | (((bits1.s6 >> 2) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \+ shared_y = sub_group_broadcast(y.s3, 2); \+ total_sums.s0 += ((((bits4.s0 & 0xF000) >> 12) | (((bits1.s2 >> 3) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \+ total_sums.s1 += ((((bits4.s1 & 0xF000) >> 12) | (((bits1.s6 >> 3) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \+ shared_y = sub_group_broadcast(y.s4, 2); \+ total_sums.s0 += (((bits4.s2 & 0x000F) | (((bits1.s2 >> 4) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \+ total_sums.s1 += (((bits4.s3 & 0x000F) | (((bits1.s6 >> 4) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \+ shared_y = sub_group_broadcast(y.s5, 2); \+ total_sums.s0 += ((((bits4.s2 & 0x00F0) >> 4) | (((bits1.s2 >> 5) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \+ total_sums.s1 += ((((bits4.s3 & 0x00F0) >> 4) | (((bits1.s6 >> 5) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \+ shared_y = sub_group_broadcast(y.s6, 2); \+ total_sums.s0 += ((((bits4.s2 & 0x0F00) >> 8) | (((bits1.s2 >> 6) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \+ total_sums.s1 += ((((bits4.s3 & 0x0F00) >> 8) | (((bits1.s6 >> 6) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \+ shared_y = sub_group_broadcast(y.s7, 2); \+ total_sums.s0 += ((((bits4.s2 & 0xF000) >> 12) | (((bits1.s2 >> 7) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \+ total_sums.s1 += ((((bits4.s3 & 0xF000) >> 12) | (((bits1.s6 >> 7) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \+ shared_y = sub_group_broadcast(y.s0, 3); \+ total_sums.s0 += (((bits4.s4 & 0x000F) | (((bits1.s3 ) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \+ total_sums.s1 += (((bits4.s5 & 0x000F) | (((bits1.s7 ) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \+ shared_y = sub_group_broadcast(y.s1, 3); \+ total_sums.s0 += ((((bits4.s4 & 0x00F0) >> 4) | (((bits1.s3 >> 1) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \+ total_sums.s1 += ((((bits4.s5 & 0x00F0) >> 4) | (((bits1.s7 >> 1) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \+ shared_y = sub_group_broadcast(y.s2, 3); \+ total_sums.s0 += ((((bits4.s4 & 0x0F00) >> 8) | (((bits1.s3 >> 2) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \+ total_sums.s1 += ((((bits4.s5 & 0x0F00) >> 8) | (((bits1.s7 >> 2) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \+ shared_y = sub_group_broadcast(y.s3, 3); \+ total_sums.s0 += ((((bits4.s4 & 0xF000) >> 12) | (((bits1.s3 >> 3) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \+ total_sums.s1 += ((((bits4.s5 & 0xF000) >> 12) | (((bits1.s7 >> 3) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \+ shared_y = sub_group_broadcast(y.s4, 3); \+ total_sums.s0 += (((bits4.s6 & 0x000F) | (((bits1.s3 >> 4) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \+ total_sums.s1 += (((bits4.s7 & 0x000F) | (((bits1.s7 >> 4) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \+ shared_y = sub_group_broadcast(y.s5, 3); \+ total_sums.s0 += ((((bits4.s6 & 0x00F0) >> 4) | (((bits1.s3 >> 5) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \+ total_sums.s1 += ((((bits4.s7 & 0x00F0) >> 4) | (((bits1.s7 >> 5) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \+ shared_y = sub_group_broadcast(y.s6, 3); \+ total_sums.s0 += ((((bits4.s6 & 0x0F00) >> 8) | (((bits1.s3 >> 6) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \+ total_sums.s1 += ((((bits4.s7 & 0x0F00) >> 8) | (((bits1.s7 >> 6) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \+ shared_y = sub_group_broadcast(y.s7, 3); \+ total_sums.s0 += ((((bits4.s6 & 0xF000) >> 12) | (((bits1.s3 >> 7) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y; \+ total_sums.s1 += ((((bits4.s7 & 0xF000) >> 12) | (((bits1.s7 >> 7) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y; \+++#define dequantizeBlockAccum_ns_q5_1_sgbroadcast_8_hi(total_sums, bits4, bits1, scale, minv, y) \+ float8 shared_y; \+ shared_y = sub_group_broadcast(y, 0); \+ total_sums.s0 += (((bits4.s0 & 0x000F) | (((bits1.s0 ) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s0; \+ total_sums.s0 += ((((bits4.s0 & 0x00F0) >> 4) | (((bits1.s0 >> 1) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s1; \+ total_sums.s0 += ((((bits4.s0 & 0x0F00) >> 8) | (((bits1.s0 >> 2) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s2; \+ total_sums.s0 += ((((bits4.s0 & 0xF000) >> 12) | (((bits1.s0 >> 3) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s3; \+ total_sums.s0 += (((bits4.s2 & 0x000F) | (((bits1.s0 >> 4) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s4; \+ total_sums.s0 += ((((bits4.s2 & 0x00F0) >> 4) | (((bits1.s0 >> 5) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s5; \+ total_sums.s0 += ((((bits4.s2 & 0x0F00) >> 8) | (((bits1.s0 >> 6) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s6; \+ total_sums.s0 += ((((bits4.s2 & 0xF000) >> 12) | (((bits1.s0 >> 7) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s7; \+ total_sums.s1 += (((bits4.s1 & 0x000F) | (((bits1.s4 ) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s0; \+ total_sums.s1 += ((((bits4.s1 & 0x00F0) >> 4) | (((bits1.s4 >> 1) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s1; \+ total_sums.s1 += ((((bits4.s1 & 0x0F00) >> 8) | (((bits1.s4 >> 2) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s2; \+ total_sums.s1 += ((((bits4.s1 & 0xF000) >> 12) | (((bits1.s4 >> 3) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s3; \+ total_sums.s1 += (((bits4.s3 & 0x000F) | (((bits1.s4 >> 4) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s4; \+ total_sums.s1 += ((((bits4.s3 & 0x00F0) >> 4) | (((bits1.s4 >> 5) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s5; \+ total_sums.s1 += ((((bits4.s3 & 0x0F00) >> 8) | (((bits1.s4 >> 6) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s6; \+ total_sums.s1 += ((((bits4.s3 & 0xF000) >> 12) | (((bits1.s4 >> 7) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s7; \+ shared_y = sub_group_broadcast(y, 1); \+ total_sums.s0 += (((bits4.s4 & 0x000F) | (((bits1.s1 ) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s0; \+ total_sums.s0 += ((((bits4.s4 & 0x00F0) >> 4) | (((bits1.s1 >> 1) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s1; \+ total_sums.s0 += ((((bits4.s4 & 0x0F00) >> 8) | (((bits1.s1 >> 2) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s2; \+ total_sums.s0 += ((((bits4.s4 & 0xF000) >> 12) | (((bits1.s1 >> 3) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s3; \+ total_sums.s0 += (((bits4.s6 & 0x000F) | (((bits1.s1 >> 4) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s4; \+ total_sums.s0 += ((((bits4.s6 & 0x00F0) >> 4) | (((bits1.s1 >> 5) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s5; \+ total_sums.s0 += ((((bits4.s6 & 0x0F00) >> 8) | (((bits1.s1 >> 6) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s6; \+ total_sums.s0 += ((((bits4.s6 & 0xF000) >> 12) | (((bits1.s1 >> 7) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s7; \+ total_sums.s1 += (((bits4.s5 & 0x000F) | (((bits1.s5 ) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s0; \+ total_sums.s1 += ((((bits4.s5 & 0x00F0) >> 4) | (((bits1.s5 >> 1) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s1; \+ total_sums.s1 += ((((bits4.s5 & 0x0F00) >> 8) | (((bits1.s5 >> 2) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s2; \+ total_sums.s1 += ((((bits4.s5 & 0xF000) >> 12) | (((bits1.s5 >> 3) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s3; \+ total_sums.s1 += (((bits4.s7 & 0x000F) | (((bits1.s5 >> 4) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s4; \+ total_sums.s1 += ((((bits4.s7 & 0x00F0) >> 4) | (((bits1.s5 >> 5) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s5; \+ total_sums.s1 += ((((bits4.s7 & 0x0F00) >> 8) | (((bits1.s5 >> 6) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s6; \+ total_sums.s1 += ((((bits4.s7 & 0xF000) >> 12) | (((bits1.s5 >> 7) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s7; \+++#define dequantizeBlockAccum_ns_q5_1_sgbroadcast_8_lo(total_sums, bits4, bits1, scale, minv, y) \+ shared_y = sub_group_broadcast(y, 2); \+ total_sums.s0 += (((bits4.s0 & 0x000F) | (((bits1.s2 ) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s0; \+ total_sums.s0 += ((((bits4.s0 & 0x00F0) >> 4) | (((bits1.s2 >> 1) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s1; \+ total_sums.s0 += ((((bits4.s0 & 0x0F00) >> 8) | (((bits1.s2 >> 2) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s2; \+ total_sums.s0 += ((((bits4.s0 & 0xF000) >> 12) | (((bits1.s2 >> 3) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s3; \+ total_sums.s0 += (((bits4.s2 & 0x000F) | (((bits1.s2 >> 4) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s4; \+ total_sums.s0 += ((((bits4.s2 & 0x00F0) >> 4) | (((bits1.s2 >> 5) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s5; \+ total_sums.s0 += ((((bits4.s2 & 0x0F00) >> 8) | (((bits1.s2 >> 6) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s6; \+ total_sums.s0 += ((((bits4.s2 & 0xF000) >> 12) | (((bits1.s2 >> 7) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s7; \+ total_sums.s1 += (((bits4.s1 & 0x000F) | (((bits1.s6 ) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s0; \+ total_sums.s1 += ((((bits4.s1 & 0x00F0) >> 4) | (((bits1.s6 >> 1) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s1; \+ total_sums.s1 += ((((bits4.s1 & 0x0F00) >> 8) | (((bits1.s6 >> 2) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s2; \+ total_sums.s1 += ((((bits4.s1 & 0xF000) >> 12) | (((bits1.s6 >> 3) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s3; \+ total_sums.s1 += (((bits4.s3 & 0x000F) | (((bits1.s6 >> 4) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s4; \+ total_sums.s1 += ((((bits4.s3 & 0x00F0) >> 4) | (((bits1.s6 >> 5) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s5; \+ total_sums.s1 += ((((bits4.s3 & 0x0F00) >> 8) | (((bits1.s6 >> 6) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s6; \+ total_sums.s1 += ((((bits4.s3 & 0xF000) >> 12) | (((bits1.s6 >> 7) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s7; \+ shared_y = sub_group_broadcast(y, 3); \+ total_sums.s0 += (((bits4.s4 & 0x000F) | (((bits1.s3 ) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s0; \+ total_sums.s0 += ((((bits4.s4 & 0x00F0) >> 4) | (((bits1.s3 >> 1) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s1; \+ total_sums.s0 += ((((bits4.s4 & 0x0F00) >> 8) | (((bits1.s3 >> 2) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s2; \+ total_sums.s0 += ((((bits4.s4 & 0xF000) >> 12) | (((bits1.s3 >> 3) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s3; \+ total_sums.s0 += (((bits4.s6 & 0x000F) | (((bits1.s3 >> 4) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s4; \+ total_sums.s0 += ((((bits4.s6 & 0x00F0) >> 4) | (((bits1.s3 >> 5) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s5; \+ total_sums.s0 += ((((bits4.s6 & 0x0F00) >> 8) | (((bits1.s3 >> 6) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s6; \+ total_sums.s0 += ((((bits4.s6 & 0xF000) >> 12) | (((bits1.s3 >> 7) & 0x01) << 4)) * scale.s0 + minv.s0) * shared_y.s7; \+ total_sums.s1 += (((bits4.s5 & 0x000F) | (((bits1.s7 ) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s0; \+ total_sums.s1 += ((((bits4.s5 & 0x00F0) >> 4) | (((bits1.s7 >> 1) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s1; \+ total_sums.s1 += ((((bits4.s5 & 0x0F00) >> 8) | (((bits1.s7 >> 2) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s2; \+ total_sums.s1 += ((((bits4.s5 & 0xF000) >> 12) | (((bits1.s7 >> 3) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s3; \+ total_sums.s1 += (((bits4.s7 & 0x000F) | (((bits1.s7 >> 4) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s4; \+ total_sums.s1 += ((((bits4.s7 & 0x00F0) >> 4) | (((bits1.s7 >> 5) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s5; \+ total_sums.s1 += ((((bits4.s7 & 0x0F00) >> 8) | (((bits1.s7 >> 6) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s6; \+ total_sums.s1 += ((((bits4.s7 & 0xF000) >> 12) | (((bits1.s7 >> 7) & 0x01) << 4)) * scale.s1 + minv.s1) * shared_y.s7; \++#ifdef ADRENO_GPU+REQD_SUBGROUP_SIZE_64+#endif+__kernel void kernel_gemv_noshuffle_q5_1_f32(+ __read_only image1d_buffer_t src0_qs, // quantized A+ global ushort * src0_qh, // 5th bits+ global half2 * src0_d, // A scales+ global half2 * src0_m, // A mins+ __read_only image1d_buffer_t src1, // B activations+ global float * dst,+ ulong offsetd,+ int ne00, // K+ int ne01) // M+{+ uint groupId = get_local_id(1);+ uint gid = get_global_id(0);+ ushort slid = get_sub_group_local_id();++ uint K = ne00;+ uint M = ne01;++ uint LINE_STRIDE_A = M / 2;+ uint BLOCK_STRIDE_A = NSUBGROUPS * M;++ __private uint4 regA;+ __private half2 regS;+ __private half2 regM;+ __private float8 regB;++ __private float2 totalSum = (float2)(0.0f);++ for (uint k = groupId; k < (K / QK5_1); k += NSUBGROUPS) {+ regS = src0_d[gid + k * LINE_STRIDE_A];+ regM = src0_m[gid + k * LINE_STRIDE_A];++ ushort4 qh_raw;+ qh_raw.s0 = src0_qh[gid + (4*k + 0) * LINE_STRIDE_A];+ qh_raw.s1 = src0_qh[gid + (4*k + 1) * LINE_STRIDE_A];+ qh_raw.s2 = src0_qh[gid + (4*k + 2) * LINE_STRIDE_A];+ qh_raw.s3 = src0_qh[gid + (4*k + 3) * LINE_STRIDE_A];++ uchar8 raw = as_uchar8(qh_raw);+ uchar8 qh_bytes = (uchar8)(raw.s0, raw.s2, raw.s4, raw.s6,+ raw.s1, raw.s3, raw.s5, raw.s7);++ // Load activations+ if (slid < 4) {+ regB.s0123 = read_imagef(src1, (slid * 2 + k * 8));+ regB.s4567 = read_imagef(src1, (1 + slid * 2 + k * 8));+ }++ regA.s0 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;+ regA.s1 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;+ regA.s2 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;+ regA.s3 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;++#ifdef VECTOR_SUB_GROUP_BROADCAST+ dequantizeBlockAccum_ns_q5_1_sgbroadcast_8_hi(totalSum, as_ushort8(regA), qh_bytes, regS, regM, regB);+#else+ dequantizeBlockAccum_ns_q5_1_sgbroadcast_1_hi(totalSum, as_ushort8(regA), qh_bytes, regS, regM, regB);+#endif // VECTOR_SUB_GROUP_BROADCAST++ regA.s0 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 4)).x;+ regA.s1 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 5)).x;+ regA.s2 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 6)).x;+ regA.s3 = read_imageui(src0_qs, (gid + k * BLOCK_STRIDE_A + LINE_STRIDE_A * 7)).x;+#ifdef VECTOR_SUB_GROUP_BROADCAST+ dequantizeBlockAccum_ns_q5_1_sgbroadcast_8_lo(totalSum, as_ushort8(regA), qh_bytes, regS, regM, regB);+#else+ dequantizeBlockAccum_ns_q5_1_sgbroadcast_1_lo(totalSum, as_ushort8(regA), qh_bytes, regS, regM, regB);+#endif // VECTOR_SUB_GROUP_BROADCAST+ }++ // reduction in local memory, assumes #wave=4+ local float2 reduceLM[SUBGROUP_SIZE * 3];+ if (groupId == 1) {+ reduceLM[SUBGROUP_SIZE * 0 + slid] = totalSum;+ }+ if (groupId == 2) {+ reduceLM[SUBGROUP_SIZE * 1 + slid] = totalSum;+ }+ if (groupId == 3) {+ reduceLM[SUBGROUP_SIZE * 2 + slid] = totalSum;+ }++ barrier(CLK_LOCAL_MEM_FENCE);++ if (groupId == 0) {+ totalSum += reduceLM[SUBGROUP_SIZE * 0 + slid];+ }+ if (groupId == 0) {+ totalSum += reduceLM[SUBGROUP_SIZE * 1 + slid];+ }+ if (groupId == 0) {+ totalSum += reduceLM[SUBGROUP_SIZE * 2 + slid];+ }++ // 2 outputs per fiber in wave 0+ if (groupId == 0) {+ dst = (global float*)((global char*)dst + offsetd);+ vstore2(totalSum, 0, &(dst[gid * 2]));+ }++}