LLM inference in C/C++
C++ ggml
opencl: initial q1_0 support (#25160)
lhezfd1a05791da1
diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txtindex 09efbc566b..52f48d122c 100644--- a/ggml/src/ggml-opencl/CMakeLists.txt+++ b/ggml/src/ggml-opencl/CMakeLists.txt@@ -78,6 +78,8 @@ set(GGML_OPENCL_KERNELSmul_mv_f16_f32_l4mul_mv_f16_f32mul_mv_f32_f32+ mul_mv_q1_0_f32+ mul_mv_q1_0_f32_flatmul_mv_q4_0_f32mul_mv_q4_0_f32_vmul_mv_q4_0_f32_8x_flat@@ -128,6 +130,7 @@ set(GGML_OPENCL_KERNELSmoe_sort_by_expertmul_mm_f32_f32_l4_lmmul_mm_f16_f32_l4_lm+ mul_mm_q1_0_f32_l4_lmmul_mm_q4_0_f32_l4_lmmul_mm_q4_1_f32_l4_lmmul_mm_q5_0_f32_l4_lm@@ -137,6 +140,8 @@ set(GGML_OPENCL_KERNELSmul_mm_q4_k_f32_l4_lmmul_mm_q5_k_f32_l4_lmmul_mm_q6_k_f32_l4_lm+ gemv_noshuffle_q1_0_f32+ gemm_noshuffle_q1_0_f32gemv_noshuffle_q4_0_f32gemv_noshuffle_q4_0_f32_specgemm_noshuffle_q4_0_f32diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cppindex 32581901b2..215aa50798 100644--- a/ggml/src/ggml-opencl/ggml-opencl.cpp+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp@@ -631,6 +631,7 @@ struct ggml_backend_opencl_context {cl_kernel kernel_mul_mm_f16_f32_kqv;cl_kernel kernel_mul_mm_f16_f32_kq;cl_kernel kernel_mul_mat_q4_0_f32, kernel_mul_mat_q4_0_f32_v;+ cl_kernel kernel_convert_block_q1_0, kernel_restore_block_q1_0;cl_kernel kernel_convert_block_q4_0, kernel_restore_block_q4_0;cl_kernel kernel_convert_block_q4_0_trans4_ns, kernel_restore_block_q4_0_trans4_ns;cl_kernel kernel_convert_block_q4_1, kernel_restore_block_q4_1;@@ -670,6 +671,7 @@ struct ggml_backend_opencl_context {cl_kernel kernel_convert_block_iq4_nl, kernel_restore_block_iq4_nl;cl_kernel kernel_convert_block_iq4_nl_noshuffle;cl_kernel kernel_restore_block_iq4_nl_noshuffle;+ cl_kernel kernel_mul_mv_q1_0_f32, kernel_mul_mv_q1_0_f32_flat;cl_kernel kernel_mul_mat_q4_0_f32_1d_8x_flat, kernel_mul_mat_q4_0_f32_1d_16x_flat;cl_kernel kernel_mul_mv_q4_1_f32;cl_kernel kernel_mul_mv_q4_1_f32_flat;@@ -733,6 +735,7 @@ struct ggml_backend_opencl_context {cl_kernel kernel_mul_mv_id_mxfp4_f32_flat;cl_kernel kernel_mul_mm_f32_f32_l4_lm;cl_kernel kernel_mul_mm_f16_f32_l4_lm;+ cl_kernel kernel_mul_mm_q1_0_f32_l4_lm;cl_kernel kernel_mul_mm_q4_0_f32_l4_lm;cl_kernel kernel_mul_mm_q4_1_f32_l4_lm;cl_kernel kernel_mul_mm_q5_0_f32_l4_lm;@@ -890,6 +893,8 @@ struct ggml_backend_opencl_context {cl_kernel kernel_gemm_noshuffle_q4_1_f32;cl_kernel kernel_gemm_noshuffle_q8_0_f32;cl_kernel kernel_gemv_noshuffle_q8_0_f32;+ cl_kernel kernel_gemm_noshuffle_q1_0_f32;+ cl_kernel kernel_gemv_noshuffle_q1_0_f32;cl_kernel kernel_gemv_noshuffle_q4_k_f32;cl_kernel kernel_gemm_noshuffle_q4_k_f32;cl_kernel kernel_gemv_noshuffle_q6_K_f32;@@ -1151,6 +1156,8 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {backend_ctx->program_cvt =build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);+ CL_CHECK((backend_ctx->kernel_convert_block_q1_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q1_0", &err), err));+ CL_CHECK((backend_ctx->kernel_restore_block_q1_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q1_0", &err), err));CL_CHECK((backend_ctx->kernel_convert_block_q4_0_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q4_0_noshuffle", &err), err));CL_CHECK((backend_ctx->kernel_restore_block_q4_0_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q4_0_noshuffle", &err), err));CL_CHECK((backend_ctx->kernel_convert_block_q4_0 = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q4_0", &err), err));@@ -1685,6 +1692,40 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {GGML_LOG_CONT(".");}+ // mul_mv_q1_0_f32+ {+#ifdef GGML_OPENCL_EMBED_KERNELS+ const std::string kernel_src {+ #include "mul_mv_q1_0_f32.cl.h"+ };+#else+ const std::string kernel_src = read_file("mul_mv_q1_0_f32.cl");+#endif+ cl_program prog =+ build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);++ CL_CHECK((backend_ctx->kernel_mul_mv_q1_0_f32 = clCreateKernel(prog, "kernel_mul_mv_q1_0_f32", &err), err));+ CL_CHECK(clReleaseProgram(prog));+ GGML_LOG_CONT(".");+ }++ // mul_mv_q1_0_f32_flat+ {+#ifdef GGML_OPENCL_EMBED_KERNELS+ const std::string kernel_src {+ #include "mul_mv_q1_0_f32_flat.cl.h"+ };+#else+ const std::string kernel_src = read_file("mul_mv_q1_0_f32_flat.cl");+#endif+ cl_program prog =+ build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);++ CL_CHECK((backend_ctx->kernel_mul_mv_q1_0_f32_flat = clCreateKernel(prog, "kernel_mul_mv_q1_0_f32_flat", &err), err));+ CL_CHECK(clReleaseProgram(prog));+ GGML_LOG_CONT(".");+ }+// mul_mv_iq4_nl_f32{#ifdef GGML_OPENCL_EMBED_KERNELS@@ -1990,6 +2031,23 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {GGML_LOG_CONT(".");}+ // mul_mm_q1_0_f32_l4_lm+ {+#ifdef GGML_OPENCL_EMBED_KERNELS+ const std::string kernel_src {+ #include "mul_mm_q1_0_f32_l4_lm.cl.h"+ };+#else+ const std::string kernel_src = read_file("mul_mm_q1_0_f32_l4_lm.cl");+#endif+ cl_program prog =+ build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);++ CL_CHECK((backend_ctx->kernel_mul_mm_q1_0_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_q1_0_f32_l4_lm", &err), err));+ CL_CHECK(clReleaseProgram(prog));+ GGML_LOG_CONT(".");+ }+// mul_mm_iq4_nl_f32_l4_lm{#ifdef GGML_OPENCL_EMBED_KERNELS@@ -2939,6 +2997,44 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {GGML_LOG_CONT(".");}+ // gemm_noshuffle_q1_0_f32+ {+#ifdef GGML_OPENCL_EMBED_KERNELS+ const std::string kernel_src {+ #include "gemm_noshuffle_q1_0_f32.cl.h"+ };+#else+ const std::string kernel_src = read_file("gemm_noshuffle_q1_0_f32.cl");+#endif+ cl_program prog = build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);+ CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q1_0_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_q1_0_f32", &err), err));+ CL_CHECK(clReleaseProgram(prog));+ GGML_LOG_CONT(".");+ }++ // gemv_noshuffle_q1_0_f32+ {+ std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std ++ " -cl-mad-enable "+ " -DSIMDGROUP_WIDTH=" ++ std::to_string(backend_ctx->adreno_wave_size);++#ifdef GGML_OPENCL_EMBED_KERNELS+ const std::string kernel_src_CL_gemv_general {+ #include "gemv_noshuffle_q1_0_f32.cl.h"+ };+#else+ const std::string kernel_src_CL_gemv_general = read_file("gemv_noshuffle_q1_0_f32.cl");+#endif++ cl_program prog = build_program_from_source(+ backend_ctx->context, backend_ctx->device, kernel_src_CL_gemv_general.c_str(), CL_gemv_compile_opts);++ CL_CHECK((backend_ctx->kernel_gemv_noshuffle_q1_0_f32 = clCreateKernel(prog, "kernel_gemv_noshuffle_q1_0_f32", &err), err));+ CL_CHECK(clReleaseProgram(prog));+ GGML_LOG_CONT(".");+ }+// gemv_noshuffle_general{std::string CL_gemv_compile_opts = std::string("-cl-std=") + opencl_c_std +@@ -4829,6 +4925,39 @@ struct ggml_tensor_extra_cl {}};+struct ggml_tensor_extra_cl_q1_0 {+ cl_mem q = nullptr;+ cl_mem q_img = nullptr;++ cl_mem d = nullptr;+ cl_mem d_img = nullptr;++ size_t size_q = 0;+ size_t size_d = 0;++ ~ggml_tensor_extra_cl_q1_0() {+ reset();+ }++ void reset() {+ // q and d are subbuffers into the bigger buffer allocated in ggml_backend_buffer.+ // They must be properly released so that the original buffer can be+ // properly released to avoid memory leak.+ if (q != nullptr) {+ CL_CHECK(clReleaseMemObject(q));+ q = nullptr;+ }+ if (d != nullptr) {+ CL_CHECK(clReleaseMemObject(d));+ d = nullptr;+ }+ q_img = nullptr;+ d_img = nullptr;+ size_q = 0;+ size_d = 0;+ }+};+// Additional tensor extra structs for quantized tensors.// These tensors are loaded from files and should not be allocated in scratch --// they should always be allocated from the pool. Hence, they do not have an@@ -5732,6 +5861,8 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_tereturn true;} else if (op->src[0]->type == GGML_TYPE_F32) {return op->src[1]->type == GGML_TYPE_F32;+ } else if (op->src[0]->type == GGML_TYPE_Q1_0) {+ return op->src[1]->type == GGML_TYPE_F32;} else if (op->src[0]->type == GGML_TYPE_Q4_0) {// Non-contig src0 routes through on-device dequant-to-f16.return op->src[1]->type == GGML_TYPE_F32;@@ -5988,6 +6119,12 @@ struct ggml_backend_opencl_buffer_context {for (ggml_tensor_extra_cl_q8_0 * e : temp_tensor_extras_q8_0_in_use) {delete e;}+ for (ggml_tensor_extra_cl_q1_0 * e : temp_tensor_extras_q1_0) {+ delete e;+ }+ for (ggml_tensor_extra_cl_q1_0 * e : temp_tensor_extras_q1_0_in_use) {+ delete e;+ }for (ggml_tensor_extra_cl_iq4_nl * e : temp_tensor_extras_iq4_nl) {delete e;}@@ -6029,6 +6166,21 @@ struct ggml_backend_opencl_buffer_context {return extra;}+ ggml_tensor_extra_cl_q1_0 * ggml_opencl_alloc_temp_tensor_extra_q1_0() {+ ggml_tensor_extra_cl_q1_0 * extra;+ if (temp_tensor_extras_q1_0.empty()) {+ extra = new ggml_tensor_extra_cl_q1_0();+ } else {+ extra = temp_tensor_extras_q1_0.back();+ temp_tensor_extras_q1_0.pop_back();+ }++ temp_tensor_extras_q1_0_in_use.push_back(extra);++ extra->reset();+ return extra;+ }+ggml_tensor_extra_cl_q4_0 * ggml_opencl_alloc_temp_tensor_extra_q4_0() {ggml_tensor_extra_cl_q4_0 * extra;if (temp_tensor_extras_q4_0.empty()) {@@ -6185,6 +6337,11 @@ struct ggml_backend_opencl_buffer_context {}temp_tensor_extras_in_use.clear();+ for (ggml_tensor_extra_cl_q1_0 * e : temp_tensor_extras_q1_0_in_use) {+ temp_tensor_extras_q1_0.push_back(e);+ }+ temp_tensor_extras_q1_0_in_use.clear();+for (ggml_tensor_extra_cl_q4_0 * e : temp_tensor_extras_q4_0_in_use) {temp_tensor_extras_q4_0.push_back(e);}@@ -6246,6 +6403,8 @@ struct ggml_backend_opencl_buffer_context {// for reuse.std::vector<ggml_tensor_extra_cl *> temp_tensor_extras;std::vector<ggml_tensor_extra_cl *> temp_tensor_extras_in_use;+ std::vector<ggml_tensor_extra_cl_q1_0 *> temp_tensor_extras_q1_0;+ std::vector<ggml_tensor_extra_cl_q1_0 *> temp_tensor_extras_q1_0_in_use;std::vector<ggml_tensor_extra_cl_q4_0 *> temp_tensor_extras_q4_0;std::vector<ggml_tensor_extra_cl_q4_0 *> temp_tensor_extras_q4_0_in_use;std::vector<ggml_tensor_extra_cl_q4_1 *> temp_tensor_extras_q4_1;@@ -6353,6 +6512,82 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,cl_command_queue queue = backend_ctx->queue;#ifdef GGML_OPENCL_SOA_Q+ if (tensor->type == GGML_TYPE_Q1_0) {+ ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra;+ GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized");++ // Allocate the new extra and create aliases from the original.+ ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context;+ ggml_tensor_extra_cl_q1_0 * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_q1_0();++ // q1_0 block = ggml_half d + (QK1_0/8) quant bytes = 2 + 16 = 18 bytes+ size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t);+ size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*(ggml_blck_size(tensor->type)/8);+ GGML_ASSERT(size_d + size_q == ggml_nbytes(tensor) && "Incorrect tensor size");++ cl_int err;+ cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE,+ ggml_nbytes(tensor), NULL, &err);+ CL_CHECK(err);+ CL_CHECK(clEnqueueWriteBuffer(+ queue, data_device, CL_TRUE, 0,+ ggml_nbytes(tensor), data, 0, NULL, NULL));++ // The original tensor memory is divided into scales and quants, i.e.,+ // we first store scales, then quants.+ cl_buffer_region region;++ // Create subbuffer for scales.+ region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment);+ region.size = size_d;+ extra->d = clCreateSubBuffer(+ extra_orig->data_device, CL_MEM_READ_WRITE,+ CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);+ CL_CHECK(err);+ auto previous_origin = region.origin;++ // Create subbuffer for quants.+ region.origin = align_to(previous_origin + size_d, backend_ctx->alignment);+ region.size = size_q;+ extra->q = clCreateSubBuffer(+ extra_orig->data_device, CL_MEM_READ_WRITE,+ CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);+ CL_CHECK(err);++ cl_kernel kernel = backend_ctx->kernel_convert_block_q1_0;++ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device));+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->q));+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->d));++ size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1};+ size_t local_work_size[] = {64, 1, 1};++ cl_event evt;+ CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt));+ CL_CHECK(clWaitForEvents(1, &evt));+ CL_CHECK(clReleaseMemObject(data_device));++ tensor->extra = extra;++ // q is uint32 (32 sign bits each); d is one half per 128-block.+#ifdef GGML_OPENCL_USE_ADRENO_KERNELS+ if (enable_adreno_trans_weight(backend_ctx, tensor)) {+ int M = tensor->ne[1]; // ne01+ int K = tensor->ne[0]; // ne00++ GGML_ASSERT(K % 128 == 0);+ GGML_ASSERT(M % 4 == 0);+ GGML_ASSERT(tensor->ne[2] == 1);+ GGML_ASSERT(tensor->ne[3] == 1);++ transpose_2d_as_32b(backend_ctx, extra->q, extra->q, size_q, K/32, M);+ transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, K/128, M);+ } // end transpose+#endif // GGML_OPENCL_USE_ADRENO_KERNELS++ return;+ }// We separate the quantized bits and scale from block_q4_0 by using an// additional kernel, where each thread handles a block. We first read the// original weights into a temporary buffer, then create two separate@@ -7743,6 +7978,63 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer,sync_with_other_backends(backend_ctx);#ifdef GGML_OPENCL_SOA_Q+ if (tensor->type == GGML_TYPE_Q1_0) {+ ggml_tensor_extra_cl_q1_0 * extra = (ggml_tensor_extra_cl_q1_0 *)tensor->extra;++#ifdef GGML_OPENCL_USE_ADRENO_KERNELS+ if (enable_adreno_trans_weight(backend_ctx, tensor)) {+ ggml_cl_buffer buf_trans_q;+ ggml_cl_buffer buf_trans_d;+ ggml_cl_buffer buf_unpacked;++ int M = tensor->ne[1];+ int K = tensor->ne[0];++ size_t size_d = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*sizeof(ggml_fp16_t);+ size_t size_q = ggml_nelements(tensor)/ggml_blck_size(tensor->type)*(ggml_blck_size(tensor->type)/8);++ buf_trans_q.allocate(backend_ctx->context, size_q);+ buf_trans_d.allocate(backend_ctx->context, size_d);+ buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor));++ transpose_2d_as_32b(backend_ctx, extra->q, buf_trans_q.buffer, size_q, M, K/32);+ transpose_2d_as_16b(backend_ctx, extra->d, buf_trans_d.buffer, size_d, M, K/128);++ cl_kernel kernel = backend_ctx->kernel_restore_block_q1_0;+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf_trans_q.buffer));+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &buf_trans_d.buffer));+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &buf_unpacked.buffer));++ size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1};+ size_t local_work_size[] = {1, 1, 1};++ cl_event evt;+ CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt));+ CL_CHECK(clWaitForEvents(1, &evt));+ CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL));+ return;+ }+#endif++ cl_int err;+ cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, ggml_nbytes(tensor), NULL, &err);+ CL_CHECK(err);++ cl_kernel kernel = backend_ctx->kernel_restore_block_q1_0;+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra->q));+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->d));+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &data_device));++ size_t global_work_size[] = {(size_t)ggml_nelements(tensor)/ggml_blck_size(tensor->type), 1, 1};+ size_t local_work_size[] = {1, 1, 1};++ cl_event evt;+ CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt));+ CL_CHECK(clWaitForEvents(1, &evt));+ CL_CHECK(clEnqueueReadBuffer(queue, data_device, CL_TRUE, offset, size, data, 0, NULL, NULL));+ CL_CHECK(clReleaseMemObject(data_device));+ return;+ }// In end-to-end runs, get_tensor is usually used to get back the logits,// where we can simply do clEnqueueReadBuffer since they are f32.// However, in test-backend-ops, the GPU graph is copied to the CPU backend,@@ -13437,7 +13729,7 @@ static void ggml_cl_mul_mat_kq_kqv_adreno(ggml_backend_t backend, const ggml_tenCL_CHECK(clReleaseMemObject(D_sub_buffer));}-static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+static void ggml_cl_mul_mat_q1_0_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {#ifdef GGML_OPENCL_USE_ADRENO_KERNELSGGML_ASSERT(src0);GGML_ASSERT(src0->extra);@@ -13446,26 +13738,34 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_tGGML_ASSERT(dst);GGML_ASSERT(dst->extra);+ GGML_ASSERT(src0->type == GGML_TYPE_Q1_0);+ GGML_ASSERT(src1->type == GGML_TYPE_F32);+ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;- ggml_tensor_extra_cl_q4_0 * extra0_q4_0 = (ggml_tensor_extra_cl_q4_0 *)src0->extra;+ ggml_tensor_extra_cl_q1_0 * extra0_q1_0 = (ggml_tensor_extra_cl_q1_0 *)src0->extra;cl_ulong offset1 = extra1->offset + src1->view_offs;cl_ulong offsetd = extrad->offset + dst->view_offs;- const int ne00 = src0->ne[0];- const int ne01 = src0->ne[1];- const int ne02 = src0->ne[2];+ GGML_ASSERT(src1->view_offs == 0);+ GGML_ASSERT(dst->view_offs == 0);- const int ne10 = src1->ne[0];- const int ne12 = src1->ne[2];+ const int ne00 = src0->ne[0];+ const int ne01 = src0->ne[1];+ const int ne02 = src0->ne[2];- const int ne0 = dst->ne[0];- const int ne1 = dst->ne[1];+ const int ne10 = src1->ne[0];+ const int ne12 = src1->ne[2];- GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0);+ const int ne0 = dst->ne[0];+ const int ne1 = dst->ne[1];++ GGML_ASSERT(ne00 == ne10);+ GGML_ASSERT((ne00 % 128) == 0);+ GGML_ASSERT(ne0 == ne01);cl_context context = backend_ctx->context;cl_kernel kernel;@@ -13484,15 +13784,204 @@ static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_tcl_mem b_sub_buf = nullptr;cl_mem b_img = nullptr;- // image for q+ // image for q (uint32: each texel packs 32 sign bits)img_fmt = { CL_R, CL_UNSIGNED_INT32};memset(&img_desc, 0, sizeof(img_desc));img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;- img_desc.image_width = M * K / 2 / 4;- img_desc.buffer = extra0_q4_0->q;+ img_desc.image_width = M * K / 32;+ img_desc.buffer = extra0_q1_0->q;CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));- // subbuffer for activations+ // create a sub_buffer for B+ region.origin = offset1;+ region.size = K * N * sizeof(float);+ CL_CHECK((b_sub_buf = clCreateSubBuffer((extra1->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));++ // image for activations+ img_fmt = {CL_RGBA, CL_FLOAT};+ memset(&img_desc, 0, sizeof(img_desc));+ img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;+ img_desc.image_width = K * N / 4;+ img_desc.buffer = b_sub_buf;+ CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));++ kernel = backend_ctx->kernel_gemv_noshuffle_q1_0_f32;++ int r2 = 1;+ int r3 = 1;++ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &q_img));+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q1_0->d));+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img));+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &extra1->offset));+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &extrad->offset));+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01));+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02));+ CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne10));+ CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12));+ CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne0));+ CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1));+ CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2));+ CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3));++ size_t wavesize = backend_ctx->adreno_wave_size;+ size_t local_work_size[] = { wavesize, 4, 1 };+ size_t global_work_size[] = { CEIL_DIV(M, wavesize)*wavesize, 4, 1 };++ backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);++ CL_CHECK(clReleaseMemObject(q_img));+ CL_CHECK(clReleaseMemObject(b_img));+ CL_CHECK(clReleaseMemObject(b_sub_buf));+ } else {+ cl_mem b_sub_buf = nullptr;+ cl_mem b_sub_buf_trans = nullptr;+ cl_mem b_img = nullptr;+ cl_mem b_img_trans = nullptr;++ // subbuffer for activations+ region.origin = offset1;+ region.size = K * N * sizeof(float);+ CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));++ // image for activations+ img_fmt = {CL_RGBA, CL_FLOAT};+ memset(&img_desc, 0, sizeof(img_desc));+ img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;+ img_desc.image_width = K * N / 4;+ img_desc.buffer = b_sub_buf;+ CL_CHECK((b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));++ // pad N to multiple of 8+ int extra_elements = N % 8;+ int padding = 0;+ if (extra_elements > 0){+ padding = 8 - extra_elements;+ }++ // subbuffer for transposed activations+ region.origin = 0;+ region.size = K * (N + padding) * sizeof(float)/2;+ backend_ctx->prealloc_act_trans.allocate(context, region.size);+ CL_CHECK((b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));++ // image for transposed activations+ img_fmt = {CL_RGBA, CL_HALF_FLOAT};+ memset(&img_desc, 0, sizeof(img_desc));+ img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;+ img_desc.image_width = K * (N + padding) / 4;+ img_desc.buffer = b_sub_buf_trans;+ CL_CHECK((b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, NULL, &err), err));++ // transpose activations+ int height_B = N/4;+ if (height_B == 0) {+ height_B = 1;+ }+ int width_B = K/4;+ int padded_height_B = (N + padding)/4;++ kernel = backend_ctx->kernel_transpose_32_16;+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img));+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans));+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_B));+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_B));+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_B));++ size_t local_work_size_t[2] = { 1, 16 };+ size_t global_work_size_t[2] = { (size_t)width_B, (size_t)padded_height_B };+ backend_ctx->enqueue_ndrange_kernel(kernel, 2, global_work_size_t, local_work_size_t, dst);++ // gemm+ kernel = backend_ctx->kernel_gemm_noshuffle_q1_0_f32;+ int padded_N = N + padding;++ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q1_0->q));+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q1_0->d));+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &b_img_trans));+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extrad->data_device));+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &K));+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(int), &M));+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &padded_N));+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &N));+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd));++ size_t global_work_size[] = { (size_t)CEIL_DIV(N, 8), (size_t)CEIL_DIV(M, 4), 1 };+ size_t local_work_size[] = { 2, 128, 1 };++ backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);++ CL_CHECK(clReleaseMemObject(b_img_trans));+ CL_CHECK(clReleaseMemObject(b_sub_buf_trans));+ CL_CHECK(clReleaseMemObject(b_img));+ CL_CHECK(clReleaseMemObject(b_sub_buf));+ }+#else+ GGML_UNUSED(backend);+ GGML_UNUSED(src0);+ GGML_UNUSED(src1);+ GGML_UNUSED(dst);+#endif+}++static void ggml_cl_mul_mat_q4_0_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+#ifdef GGML_OPENCL_USE_ADRENO_KERNELS+ GGML_ASSERT(src0);+ GGML_ASSERT(src0->extra);+ GGML_ASSERT(src1);+ GGML_ASSERT(src1->extra);+ GGML_ASSERT(dst);+ GGML_ASSERT(dst->extra);++ ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;++ ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra;+ ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;+ ggml_tensor_extra_cl_q4_0 * extra0_q4_0 = (ggml_tensor_extra_cl_q4_0 *)src0->extra;++ cl_ulong offset1 = extra1->offset + src1->view_offs;+ cl_ulong offsetd = extrad->offset + dst->view_offs;++ const int ne00 = src0->ne[0];+ const int ne01 = src0->ne[1];+ const int ne02 = src0->ne[2];++ const int ne10 = src1->ne[0];+ const int ne12 = src1->ne[2];++ const int ne0 = dst->ne[0];+ const int ne1 = dst->ne[1];++ GGML_ASSERT(ne00 % ggml_blck_size(src0->type) == 0);++ cl_context context = backend_ctx->context;+ cl_kernel kernel;++ cl_int err;+ cl_image_format img_fmt;+ cl_image_desc img_desc;+ cl_buffer_region region;++ int M = ne01;+ int N = ne1;+ int K = ne00;++ if (ne1 == 1) {+ cl_mem q_img = nullptr;+ cl_mem b_sub_buf = nullptr;+ cl_mem b_img = nullptr;++ // image for q+ img_fmt = { CL_R, CL_UNSIGNED_INT32};+ memset(&img_desc, 0, sizeof(img_desc));+ img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;+ img_desc.image_width = M * K / 2 / 4;+ img_desc.buffer = extra0_q4_0->q;+ CL_CHECK((q_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));++ // subbuffer for activationsregion.origin = offset1;region.size = K * N * sizeof(float);CL_CHECK((b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));@@ -15311,6 +15800,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co// view->extra stays pre-SoA; cast to the SoA struct would SIGSEGV.// Follow view_src to reach the real SoA extra.const ggml_tensor * soa0_src = src0->view_src != nullptr ? src0->view_src : src0;+ ggml_tensor_extra_cl_q1_0 * extra0_q1_0 = (ggml_tensor_extra_cl_q1_0 *)src0->extra;ggml_tensor_extra_cl_q4_0 * extra0_q4_0 = (ggml_tensor_extra_cl_q4_0 *)soa0_src->extra;ggml_tensor_extra_cl_q4_1 * extra0_q4_1 = (ggml_tensor_extra_cl_q4_1 *)soa0_src->extra;ggml_tensor_extra_cl_q5_0 * extra0_q5_0 = (ggml_tensor_extra_cl_q5_0 *)soa0_src->extra;@@ -15374,6 +15864,13 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co// a limit check, but q4_0 / q4_1 tensors are very unlikely to exceed that// limit, so the check is omitted.+ // q1_0 x fp32+ if (src0t == GGML_TYPE_Q1_0 && src1t == GGML_TYPE_F32 &&+ enable_adreno_trans_weight(backend_ctx, src0)) {+ ggml_cl_mul_mat_q1_0_f32_adreno(backend, src0, src1, dst);+ return;+ }+// q4_0 x fp32if(src0t == GGML_TYPE_Q4_0 && src1t == GGML_TYPE_F32) {ggml_cl_mul_mat_q4_0_f32_adreno(backend, src0, src1, dst);@@ -15577,6 +16074,48 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, cobackend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);return;}+ case GGML_TYPE_Q1_0: {+ if (ne11 < 32) {+ break;+ }+ if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) {+ break;+ }++ kernel = backend_ctx->kernel_mul_mm_q1_0_f32_l4_lm;+ nth0 = 128; // calculated as (BM*BN)/(TM*TN)++ int batch_stride_a = ne00*ne01;+ int batch_stride_b = ne10*ne11;+ int batch_stride_d = ne0*ne1;++ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q1_0->q));+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q1_0->d));+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01));+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &ne02));+ CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne11));+ CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne12));+ CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne10)); // stride_a+ CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne10)); // stride_b+ CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne01)); // stride_d+ CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &batch_stride_a));+ CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &batch_stride_b));+ CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &batch_stride_d));+ CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2));+ CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3));++ // 64 is block tile size BM and BN - change here when BM and BN in the kernel are changed.+ size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)(CEIL_DIV(ne11, 64)), (size_t)ne12*ne13};+ size_t local_work_size[] = {(size_t)nth0, 1, 1};++ backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);+ return;+ }case GGML_TYPE_Q4_0: {if (ne11 < 32) {break;@@ -16165,6 +16704,81 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, coCL_CHECK(clSetKernelArg(kernel, 22, sizeof(int), &r2));CL_CHECK(clSetKernelArg(kernel, 23, sizeof(int), &r3));break;+ case GGML_TYPE_Q1_0: {+#ifdef GGML_OPENCL_SOA_Q+ kernel = backend_ctx->kernel_mul_mv_q1_0_f32_flat;++ // nth0 - subgroup size+ // nth1 - number of subgroups per workgroup+ // ndst - number of output values per workgroup = output per subgroup * number of subgroups+ if (backend_ctx->gpu_family == INTEL) {+ nth0 = 16;+ nth1 = 2;+ ndst = nth1*4;+ } else if (backend_ctx->gpu_family == ADRENO) {+ nth0 = 64;+ nth1 = 2;+ ndst = nth1*4;+ } else {+ GGML_ASSERT(false && "TODO: Unknown GPU");+ }++ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_q1_0->q));+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_q1_0->d));+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01));+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01));+ CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02));+ CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03));+ CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12));+ CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11));+ CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12));+ CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13));+ CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0));+ CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1));+ CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2));+ CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3));+#else+ kernel = backend_ctx->kernel_mul_mv_q1_0_f32;++ if (backend_ctx->gpu_family == INTEL) {+ nth0 = 16;+ nth1 = 2;+ ndst = nth1*4;+ } else if (backend_ctx->gpu_family == ADRENO) {+ nth0 = 64;+ nth1 = 2;+ ndst = nth1*4;+ } else {+ GGML_ASSERT(false && "TODO: Unknown GPU");+ }++ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device));+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1));+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device));+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd));+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01));+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01));+ CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02));+ CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03));+ CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12));+ CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11));+ CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12));+ CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13));+ CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0));+ CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1));+ CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2));+ CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3));+#endif // GGML_OPENCL_SOA_Q+ break;+ }case GGML_TYPE_Q4_0:// This should have been satisfied.GGML_ASSERT(ne11 == ne1);@@ -16879,6 +17493,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, cosrc0t == GGML_TYPE_Q5_0 ||src0t == GGML_TYPE_Q5_1 ||src0t == GGML_TYPE_Q8_0 ||+ src0t == GGML_TYPE_Q1_0 ||src0t == GGML_TYPE_IQ4_NL ||src0t == GGML_TYPE_Q2_K) {// Each SIMD group produces N_DST values in the result. Assuming eachdiff --git a/ggml/src/ggml-opencl/kernels/cvt.cl b/ggml/src/ggml-opencl/kernels/cvt.clindex 82a1305592..bf0335a698 100644--- a/ggml/src/ggml-opencl/kernels/cvt.cl+++ b/ggml/src/ggml-opencl/kernels/cvt.cl@@ -27,6 +27,8 @@#define QR5_1 2#define QK8_0 32#define QR8_0 1+#define QK1_0 128+#define QR1_0 1#define QK_K 256#define K_SCALE_SIZE (3 * QK_K / 64)#define K_QUANTS_PER_ITERATION 2@@ -38,6 +40,14 @@ typedef ushort uint16_t;typedef int int32_t;typedef uint uint32_t;+//------------------------------------------------------------------------------+// block_q1_0+//------------------------------------------------------------------------------+typedef struct {+ half d; // delta+ uchar qs[QK1_0/8]; // 1-bit signs (16 bytes)+} block_q1_0;+//------------------------------------------------------------------------------// block_q4_0//------------------------------------------------------------------------------@@ -159,6 +169,42 @@ kernel void kernel_convert_f16_to_bf16(}}+//------------------------------------------------------------------------------+// kernel_convert_block_q1_0+// Convert block_q1_0 (AOS) to 2 separate arrays (SOA): quant bytes + scales.+// q1_0 bits are stored in natural order (bit j of byte i -> weight 8*i + j)+//------------------------------------------------------------------------------+kernel void kernel_convert_block_q1_0(+ global block_q1_0 * src0,+ global uchar * dst_q,+ global half * dst_d+) {+ global block_q1_0 * b = (global block_q1_0 *) src0 + get_global_id(0);+ global uchar * q = (global uchar *) dst_q + (QK1_0/8)*get_global_id(0);+ global half * d = (global half *) dst_d + get_global_id(0);++ *d = b->d;++ for (int i = 0; i < QK1_0/8; ++i) {+ q[i] = b->qs[i];+ }+}++kernel void kernel_restore_block_q1_0(+ global uchar * src_q,+ global half * src_d,+ global block_q1_0 * dst+) {+ global block_q1_0 * b = (global block_q1_0 *) dst + get_global_id(0);+ global uchar * q = (global uchar *) src_q + (QK1_0/8)*get_global_id(0);+ global half * d = (global half *) src_d + get_global_id(0);++ b->d = *d;+ for (int i = 0; i < QK1_0/8; ++i) {+ b->qs[i] = q[i];+ }+}+//------------------------------------------------------------------------------// kernel_convert_block_q4_0// Convert the block_q4_0 format to 2 separate arrays (AOS -> SOA).diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q1_0_f32.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q1_0_f32.clnew file mode 100644index 0000000000..ba46cc75c9--- /dev/null+++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_q1_0_f32.cl@@ -0,0 +1,94 @@+#pragma OPENCL EXTENSION cl_khr_fp16 : enable+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable++#ifdef cl_qcom_reqd_sub_group_size+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable+#define ADRENO_GPU 1+#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))+#endif++// each work-item computes a 4 (rows of A / m) x 8 (cols of B / n) output tile.+#ifdef ADRENO_GPU+REQD_SUBGROUP_SIZE_128+#endif+kernel void kernel_gemm_noshuffle_q1_0_f32(+ global const uint * src0_q,+ global const half * src0_d,+ read_only image1d_buffer_t src1,+ global float * dst,+ int k,+ int m,+ int n,+ int n_no_padding,+ ulong offsetd+) {+ int n_4 = n >> 2;++ int gy = get_global_id(0);+ int gx = get_global_id(1);+ int gx_2 = gx << 2;+ dst = (global float *)((global char*)dst + offsetd);++ half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0;+ half8 B;++ global const uint* wptr = src0_q + gx_2;+ global const half* sptr = src0_d + gx_2;++ // 32 weights per uint32, 128 weights (one block / one scale) per 4 uint32.+ for (int i = 0; i < k; i += 32) {+ uint4 pack4 = vload4(0, wptr + (i / 32) * m); // 4 rows, 32 K-values each+ half4 scale = vload4(0, sptr + (i / 128) * m); // 4 rows, one scale per 128++ for (int j = 0; j < 32; ++j) {+ B.s0123 = read_imageh(src1, gy * 2 + (i + j) * n_4);+ B.s4567 = read_imageh(src1, gy * 2 + (i + j) * n_4 + 1);++ // sign bit -> +-1 (half arithmetic avoids unsigned underflow)+ half4 wj = (half4)(+ 2.0h * (half)((pack4.s0 >> j) & 1u) - 1.0h,+ 2.0h * (half)((pack4.s1 >> j) & 1u) - 1.0h,+ 2.0h * (half)((pack4.s2 >> j) & 1u) - 1.0h,+ 2.0h * (half)((pack4.s3 >> j) & 1u) - 1.0h) * scale;++ c0 += B * wj.s0;+ c1 += B * wj.s1;+ c2 += B * wj.s2;+ c3 += B * wj.s3;+ }+ }++ int idx = (gy << 3) * m + (gx << 2);++ if(idx+3 < m*n_no_padding){+ vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx);+ idx += m;+ }+ if(idx+3 < m*n_no_padding){+ vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx);+ idx += m;+ }+ if(idx+3 < m*n_no_padding){+ vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx);+ idx += m;+ }+ if(idx+3 < m*n_no_padding){+ vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx);+ idx += m;+ }+ if(idx+3 < m*n_no_padding){+ vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx);+ idx += m;+ }+ if(idx+3 < m*n_no_padding){+ vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx);+ idx += m;+ }+ if(idx+3 < m*n_no_padding){+ vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx);+ idx += m;+ }+ if(idx+3 < m*n_no_padding){+ vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx);+ }+}diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.clnew file mode 100644index 0000000000..e83c5d0689--- /dev/null+++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q1_0_f32.cl@@ -0,0 +1,121 @@+#pragma OPENCL EXTENSION cl_khr_fp16 : enable+#pragma OPENCL EXTENSION cl_khr_subgroups : enable++#ifdef cl_qcom_reqd_sub_group_size+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable+#define ADRENO_GPU 1+#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))+#endif++#define QK1_0 128+#define N_SIMDGROUP 4++#define dequantizeBlockAccum_q1(total, bits, scale, regB, lb) \+ total += (2.0f*(float)((bits >> 0) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s0, lb+0); \+ total += (2.0f*(float)((bits >> 1) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s1, lb+0); \+ total += (2.0f*(float)((bits >> 2) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s2, lb+0); \+ total += (2.0f*(float)((bits >> 3) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s3, lb+0); \+ total += (2.0f*(float)((bits >> 4) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s4, lb+0); \+ total += (2.0f*(float)((bits >> 5) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s5, lb+0); \+ total += (2.0f*(float)((bits >> 6) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s6, lb+0); \+ total += (2.0f*(float)((bits >> 7) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s7, lb+0); \+ total += (2.0f*(float)((bits >> 8) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s0, lb+1); \+ total += (2.0f*(float)((bits >> 9) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s1, lb+1); \+ total += (2.0f*(float)((bits >> 10) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s2, lb+1); \+ total += (2.0f*(float)((bits >> 11) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s3, lb+1); \+ total += (2.0f*(float)((bits >> 12) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s4, lb+1); \+ total += (2.0f*(float)((bits >> 13) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s5, lb+1); \+ total += (2.0f*(float)((bits >> 14) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s6, lb+1); \+ total += (2.0f*(float)((bits >> 15) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s7, lb+1); \+ total += (2.0f*(float)((bits >> 16) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s0, lb+2); \+ total += (2.0f*(float)((bits >> 17) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s1, lb+2); \+ total += (2.0f*(float)((bits >> 18) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s2, lb+2); \+ total += (2.0f*(float)((bits >> 19) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s3, lb+2); \+ total += (2.0f*(float)((bits >> 20) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s4, lb+2); \+ total += (2.0f*(float)((bits >> 21) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s5, lb+2); \+ total += (2.0f*(float)((bits >> 22) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s6, lb+2); \+ total += (2.0f*(float)((bits >> 23) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s7, lb+2); \+ total += (2.0f*(float)((bits >> 24) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s0, lb+3); \+ total += (2.0f*(float)((bits >> 25) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s1, lb+3); \+ total += (2.0f*(float)((bits >> 26) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s2, lb+3); \+ total += (2.0f*(float)((bits >> 27) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s3, lb+3); \+ total += (2.0f*(float)((bits >> 28) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s4, lb+3); \+ total += (2.0f*(float)((bits >> 29) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s5, lb+3); \+ total += (2.0f*(float)((bits >> 30) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s6, lb+3); \+ total += (2.0f*(float)((bits >> 31) & 1u) - 1.0f) * scale * sub_group_broadcast(regB.s7, lb+3);+++#ifdef ADRENO_GPU+REQD_SUBGROUP_SIZE_64+#endif+__kernel void kernel_gemv_noshuffle_q1_0_f32(+ read_only image1d_buffer_t src0_q,+ global half * src0_d,+ read_only image1d_buffer_t src1,+ ulong offset1,+ global float * dst,+ ulong offsetd,+ int ne00,+ int ne01,+ int ne02,+ int ne10,+ int ne12,+ int ne0,+ int ne1,+ int r2,+ int r3)+{+ uint groupId = get_local_id(1);+ uint gid = get_global_id(0);+ ushort slid = get_sub_group_local_id();++ uint K = ne00;+ uint M = ne01;++ uint LINE_STRIDE_A = M;+ uint BLOCK_STRIDE_A = 4 * M;++ uint4 regA;+ half regS;+ float8 regB;++ float totalSum = 0.0f;++ #pragma unroll 1+ for (uint kb = groupId; kb < (K / QK1_0); kb += N_SIMDGROUP) {+ regS = src0_d[gid + kb * LINE_STRIDE_A]; // each fiber loads its row's scale++ // first 16 fibers load 8 B values each -> 128 activations for this block+ if (slid < 16) {+ regB.s0123 = read_imagef(src1, (slid * 2 + kb * 32));+ regB.s4567 = read_imagef(src1, (1 + slid * 2 + kb * 32));+ }++ // load this row's 4 uint32 (128 sign bits)+ regA.s0 = read_imageui(src0_q, (gid + kb * BLOCK_STRIDE_A + LINE_STRIDE_A * 0)).x;+ regA.s1 = read_imageui(src0_q, (gid + kb * BLOCK_STRIDE_A + LINE_STRIDE_A * 1)).x;+ regA.s2 = read_imageui(src0_q, (gid + kb * BLOCK_STRIDE_A + LINE_STRIDE_A * 2)).x;+ regA.s3 = read_imageui(src0_q, (gid + kb * BLOCK_STRIDE_A + LINE_STRIDE_A * 3)).x;++ float scale = (float)regS;+ dequantizeBlockAccum_q1(totalSum, regA.s0, scale, regB, 0);+ dequantizeBlockAccum_q1(totalSum, regA.s1, scale, regB, 4);+ dequantizeBlockAccum_q1(totalSum, regA.s2, scale, regB, 8);+ dequantizeBlockAccum_q1(totalSum, regA.s3, scale, regB, 12);+ }++ // reduction in local memory, assumes #wave = N_SIMDGROUP = 4+ local float reduceLM[SIMDGROUP_WIDTH * 3];+ if (groupId == 1) reduceLM[SIMDGROUP_WIDTH * 0 + slid] = totalSum;+ if (groupId == 2) reduceLM[SIMDGROUP_WIDTH * 1 + slid] = totalSum;+ if (groupId == 3) reduceLM[SIMDGROUP_WIDTH * 2 + slid] = totalSum;+ barrier(CLK_LOCAL_MEM_FENCE);+ if (groupId == 0) totalSum += reduceLM[SIMDGROUP_WIDTH * 0 + slid];+ if (groupId == 0) totalSum += reduceLM[SIMDGROUP_WIDTH * 1 + slid];+ if (groupId == 0) totalSum += reduceLM[SIMDGROUP_WIDTH * 2 + slid];++ if (groupId == 0) {+ dst = (global float*)((global char*)dst + offsetd);+ dst[gid] = totalSum;+ }+}diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_q1_0_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_q1_0_f32_l4_lm.clnew file mode 100644index 0000000000..67710fac4b--- /dev/null+++ b/ggml/src/ggml-opencl/kernels/mul_mm_q1_0_f32_l4_lm.cl@@ -0,0 +1,156 @@+#pragma OPENCL EXTENSION cl_khr_fp16 : enable++// LOAD_VEC_A is 8 because one q1_0 quant byte expands to 8 weights along K.+#define LOAD_VEC_A 8+#define LOAD_VEC_B 4++#define BM 64+#define BN 64+#define BK 32+#define TM 4+#define TN 8++kernel void kernel_mul_mm_q1_0_f32_l4_lm(+ global uchar * src0_q,+ global half * src0_d,+ global float4 * src1,+ ulong offset1,+ global float * dst,+ ulong offsetd,++ int ne00,+ int ne01,+ int ne02,+ int ne11,+ int ne12,++ int stride_a,+ int stride_b,+ int stride_d,++ int batch_stride_a,+ int batch_stride_b,+ int batch_stride_d,++ int r2,+ int r3+) {+ src1 = (global float4*)((global char*)src1 + offset1);+ dst = (global float *)((global char*)dst + offsetd);++ local float buf_a[BM * BK];+ local float buf_b[BN * BK];++ const int batch_idx = get_global_id(2);++ const int i13 = batch_idx / ne12;+ const int i12 = batch_idx % ne12;++ const int i03 = i13 / r3;+ const int i02 = i12 / r2;++ const int batch_idx_a = i03 * ne02 + i02;++ const int ir = get_group_id(0);+ const int ic = get_group_id(1);++ const int tid = get_local_id(0);+ const int th_r = tid % (BM / TM);+ const int th_c = tid / (BM / TM);++ const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A);+ const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A);+ const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B);+ const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B);++ const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK;+ const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK;++ int pos_a = (batch_idx_a * batch_stride_a + ir * BM * stride_a) / LOAD_VEC_A;+ int pos_b = (batch_idx * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B;++ float sums[TM * TN];+ float cache_a[TM];+ float cache_b[TN];++ for (int i = 0; i < TM * TN; i++) {+ sums[i] = 0.0f;+ }++ for (int block = 0; block < ne00; block += BK) {+ for (int l = 0; l < BM; l += loadstride_a) {+ if (ir*BM + loadc_a + l < ne01) {+ int idx = pos_a + (loadc_a + l) * stride_a / LOAD_VEC_A + loadr_a;+ int ib = idx / 16; // 16 quant bytes per q1_0 block++ float d = (float)src0_d[ib];+ uint bits = src0_q[idx];++ // use float to avoid unsigned underflow of (2*0 - 1).+ buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = d * (2.0f*(float)((bits >> 0) & 1) - 1.0f);+ buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = d * (2.0f*(float)((bits >> 1) & 1) - 1.0f);+ buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = d * (2.0f*(float)((bits >> 2) & 1) - 1.0f);+ buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = d * (2.0f*(float)((bits >> 3) & 1) - 1.0f);+ buf_a[(loadr_a * LOAD_VEC_A + 4) * BM + loadc_a + l] = d * (2.0f*(float)((bits >> 4) & 1) - 1.0f);+ buf_a[(loadr_a * LOAD_VEC_A + 5) * BM + loadc_a + l] = d * (2.0f*(float)((bits >> 5) & 1) - 1.0f);+ buf_a[(loadr_a * LOAD_VEC_A + 6) * BM + loadc_a + l] = d * (2.0f*(float)((bits >> 6) & 1) - 1.0f);+ buf_a[(loadr_a * LOAD_VEC_A + 7) * BM + loadc_a + l] = d * (2.0f*(float)((bits >> 7) & 1) - 1.0f);+ } else {+ for (int b = 0; b < LOAD_VEC_A; ++b) {+ buf_a[(loadr_a * LOAD_VEC_A + b) * BM + loadc_a + l] = 0.0f;+ }+ }+ }++ for (int l = 0; l < BN; l += loadstride_b) {+ if (ic*BN + loadc_b + l < ne11) {+ int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b;+ buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0;+ buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1;+ buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2;+ buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3;+ } else {+ buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f;+ buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f;+ buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f;+ buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f;+ }+ }++ barrier(CLK_LOCAL_MEM_FENCE);++ pos_a += BK / LOAD_VEC_A;+ pos_b += BK / LOAD_VEC_B;++ for (int i = 0; i < BK; i++) {+ for (int j = 0; j < TM; j++) {+ cache_a[j] = buf_a[(i) * BM + th_r * TM + j];+ }++ for (int j = 0; j < TN; j++) {+ cache_b[j] = buf_b[(i) * BN + th_c * TN + j];+ }++ for (int cc = 0; cc < TN; cc++) {+ for (int cr = 0; cr < TM; cr++) {+ const int sums_idx = cc*TM + cr;+ sums[sums_idx] = mad(cache_a[cr], cache_b[cc], sums[sums_idx]);+ }+ }+ }+ barrier(CLK_LOCAL_MEM_FENCE);+ }++ const int dr = ir * BM + th_r * TM;+ const int dc = ic * BN + th_c * TN;++ const int offsets = batch_idx * batch_stride_d;++ for (int cc = 0; cc < TN; cc++) {+ for (int cr = 0; cr < TM; cr++) {+ if (dr + cr < ne01 && dc + cc < ne11) {+ dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr];+ }+ }+ }+}diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32.clnew file mode 100644index 0000000000..8746e2a3aa--- /dev/null+++ b/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32.cl@@ -0,0 +1,141 @@+#pragma OPENCL EXTENSION cl_khr_fp16 : enable++#ifdef cl_intel_subgroups+#pragma OPENCL EXTENSION cl_intel_subgroups : enable+#else+#pragma OPENCL EXTENSION cl_khr_subgroups : enable+#endif++#ifdef cl_intel_required_subgroup_size+#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable+#define INTEL_GPU 1+#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))+#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))+#elif defined(cl_qcom_reqd_sub_group_size)+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable+#define ADRENO_GPU 1+#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))+#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))+#endif++#define QK1_0 128+typedef struct {+ half d;+ uchar qs[QK1_0/8];+} block_q1_0;++#define NB_Q1_0 16++#ifdef INTEL_GPU+#define N_R0_Q1_0 4 // number of rows each subgroup works on+#define N_SG_Q1_0 2 // number of subgroups in a work group+#define N_SIMDWIDTH 16 // subgroup size+#elif defined (ADRENO_GPU)+#define N_R0_Q1_0 4+#define N_SG_Q1_0 2+#define N_SIMDWIDTH 64+#endif++inline float block_q_1_0_dot_y(global block_q1_0 * qb, float sumy, float yl[NB_Q1_0], short il) {+ global uchar * qs = qb->qs + il*2;+ uint b0 = qs[0];+ uint b1 = qs[1];++ float acc = 0.f;+ acc += yl[ 0]*(float)((b0 >> 0) & 1) + yl[ 1]*(float)((b0 >> 1) & 1);+ acc += yl[ 2]*(float)((b0 >> 2) & 1) + yl[ 3]*(float)((b0 >> 3) & 1);+ acc += yl[ 4]*(float)((b0 >> 4) & 1) + yl[ 5]*(float)((b0 >> 5) & 1);+ acc += yl[ 6]*(float)((b0 >> 6) & 1) + yl[ 7]*(float)((b0 >> 7) & 1);++ acc += yl[ 8]*(float)((b1 >> 0) & 1) + yl[ 9]*(float)((b1 >> 1) & 1);+ acc += yl[10]*(float)((b1 >> 2) & 1) + yl[11]*(float)((b1 >> 3) & 1);+ acc += yl[12]*(float)((b1 >> 4) & 1) + yl[13]*(float)((b1 >> 5) & 1);+ acc += yl[14]*(float)((b1 >> 6) & 1) + yl[15]*(float)((b1 >> 7) & 1);++ return qb->d * (2.0f*acc - sumy);+}++#ifdef INTEL_GPU+REQD_SUBGROUP_SIZE_16+#elif defined (ADRENO_GPU)+REQD_SUBGROUP_SIZE_64+#endif+kernel void kernel_mul_mv_q1_0_f32(+ global char * src0,+ ulong offset0,+ global char * src1,+ ulong offset1,+ global char * dst,+ ulong offsetd,+ int ne00,+ int ne01,+ ulong nb01,+ ulong nb02,+ ulong nb03,+ int ne12,+ ulong nb11,+ ulong nb12,+ ulong nb13,+ int ne0,+ int ne1,+ int r2,+ int r3+) {+ src0 = (global char*)((global char*)src0 + offset0);+ src1 = (global char*)((global char*)src1 + offset1);+ dst = (global char*)((global char*)dst + offsetd);++ int nb = ne00/QK1_0;++ int r0 = get_group_id(0);+ int r1 = get_group_id(1);+ int im = get_group_id(2);++ int first_row = (r0*N_SG_Q1_0 + get_sub_group_id()) * N_R0_Q1_0;++ uint i12 = im%ne12;+ uint i13 = im/ne12;++ ulong offset_src1 = r1*nb11 + i12*nb12 + i13*nb13;+ global float * y = (global float *) (src1 + offset_src1);++ // pointers to src0 rows+ global block_q1_0 * ax[N_R0_Q1_0];+ for (int row = 0; row < N_R0_Q1_0; ++row) {+ ulong offset_src0 = (first_row + row)*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;+ ax[row] = (global block_q1_0 *) ((global char *) src0 + offset_src0);+ }++ float yl[NB_Q1_0];+ float sumf[N_R0_Q1_0] = { 0.f };++ const short ix = get_sub_group_local_id()/8;+ const short il = get_sub_group_local_id()%8;++ global float * yb = y + ix*QK1_0 + il*NB_Q1_0;++ // each thread handles NB_Q1_0 quants at a time+ for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/8) {+ float sumy = 0.f;+ for (short i = 0; i < NB_Q1_0; ++i) {+ yl[i] = yb[i];+ sumy += yb[i];+ }++ for (short row = 0; row < N_R0_Q1_0; row++) {+ sumf[row] += block_q_1_0_dot_y(ax[row] + ib, sumy, yl, il);+ }++ yb += N_SIMDWIDTH*NB_Q1_0;+ }++ global float * dst_f32 = (global float *) dst + (ulong)im*ne0*ne1 + (ulong)r1*ne0;++ for (int row = 0; row < N_R0_Q1_0; ++row) {+ float tot = sub_group_reduce_add(sumf[row]);++ if (get_sub_group_local_id() == 0 && first_row + row < ne01) {+ dst_f32[first_row + row] = tot;+ }+ }+}diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32_flat.clnew file mode 100644index 0000000000..870e36d866--- /dev/null+++ b/ggml/src/ggml-opencl/kernels/mul_mv_q1_0_f32_flat.cl@@ -0,0 +1,190 @@+#pragma OPENCL EXTENSION cl_khr_fp16 : enable++#ifdef cl_intel_subgroups+#pragma OPENCL EXTENSION cl_intel_subgroups : enable+#else+#pragma OPENCL EXTENSION cl_khr_subgroups : enable+#endif++#ifdef cl_intel_required_subgroup_size+#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable+#define INTEL_GPU 1+#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))+#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))+#elif defined(cl_qcom_reqd_sub_group_size)+#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable+#define ADRENO_GPU 1+#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half")))+#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))+#endif++#define QK1_0 128+#define QK1_0_BYTES (QK1_0/8) // 16 quant bytes per block+#define QK1_0_BLK_BYTES (QK1_0_BYTES + 2) // d + qs in original tensor = 18++#define NB_Q1_0 16 // quants handled per thread (two qs bytes)++#ifdef INTEL_GPU+#define N_R0_Q1_0 4 // number of rows each subgroup works on+#define N_SG_Q1_0 2 // number of subgroups in a work group+#define N_SIMDWIDTH 16 // subgroup size+#elif defined (ADRENO_GPU)+#define N_R0_Q1_0 4+#define N_SG_Q1_0 2+#define N_SIMDWIDTH 64+#endif++#ifdef INTEL_GPU+REQD_SUBGROUP_SIZE_16+#elif defined (ADRENO_GPU)+REQD_SUBGROUP_SIZE_64+#endif+kernel void kernel_mul_mv_q1_0_f32_flat(+ global char * src0_q,+ global half * src0_d,+ global char * src1,+ ulong offset1,+ global char * dst,+ ulong offsetd,+ int ne00,+ int ne01,+ ulong nb01,+ ulong nb02,+ ulong nb03,+ int ne12,+ ulong nb11,+ ulong nb12,+ ulong nb13,+ int ne0,+ int ne1,+ int r2,+ int r3+) {+ src1 = (global char*)((global char*)src1 + offset1);+ dst = (global char*)((global char*)dst + offsetd);++ int nb = ne00/QK1_0;++ int r0 = get_group_id(0);+ int r1 = get_group_id(1);+ int im = get_group_id(2);++ int first_row = (r0*N_SG_Q1_0 + get_sub_group_id()) * N_R0_Q1_0;++ uint i12 = im%ne12;+ uint i13 = im/ne12;++ ulong offset_src1 = r1*nb11 + i12*nb12 + i13*nb13;+ global float * y = (global float *) (src1 + offset_src1);++ // pointers to src0 rows (flat: q bytes + scales)+ uint offset_src0_base = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03;++ global uchar * ax0, * ax1, * ax2, * ax3;+ global half * ad0, * ad1, * ad2, * ad3;+ uint offset_src0;++ offset_src0 = (offset_src0_base + 0*nb01) / QK1_0_BLK_BYTES;+ ax0 = (global uchar *) ((global char *) src0_q + offset_src0*QK1_0_BYTES);+ ad0 = (global half *) ((global char *) src0_d + offset_src0*sizeof(half));++ offset_src0 = (offset_src0_base + 1*nb01) / QK1_0_BLK_BYTES;+ ax1 = (global uchar *) ((global char *) src0_q + offset_src0*QK1_0_BYTES);+ ad1 = (global half *) ((global char *) src0_d + offset_src0*sizeof(half));++ offset_src0 = (offset_src0_base + 2*nb01) / QK1_0_BLK_BYTES;+ ax2 = (global uchar *) ((global char *) src0_q + offset_src0*QK1_0_BYTES);+ ad2 = (global half *) ((global char *) src0_d + offset_src0*sizeof(half));++ offset_src0 = (offset_src0_base + 3*nb01) / QK1_0_BLK_BYTES;+ ax3 = (global uchar *) ((global char *) src0_q + offset_src0*QK1_0_BYTES);+ ad3 = (global half *) ((global char *) src0_d + offset_src0*sizeof(half));++ const short ix = get_sub_group_local_id()/8;+ const short il = get_sub_group_local_id()%8;++ global float * yb = y + ix*QK1_0 + il*NB_Q1_0;++ float8 yl_lo;+ float8 yl_hi;+ float4 sumf = 0.f;++ // each thread handles NB_Q1_0 = 16 quants (two qs bytes) at a time+ for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/8) {+ yl_lo = vload8(0, yb);+ yl_hi = vload8(0, yb + 8);+ float sumy = yl_lo.s0 + yl_lo.s1 + yl_lo.s2 + yl_lo.s3+ + yl_lo.s4 + yl_lo.s5 + yl_lo.s6 + yl_lo.s7+ + yl_hi.s0 + yl_hi.s1 + yl_hi.s2 + yl_hi.s3+ + yl_hi.s4 + yl_hi.s5 + yl_hi.s6 + yl_hi.s7;++ uint b0, b1;+ float acc;++ b0 = ax0[ib*QK1_0_BYTES + il*2 + 0];+ b1 = ax0[ib*QK1_0_BYTES + il*2 + 1];+ acc = yl_lo.s0*(float)((b0 >> 0) & 1) + yl_lo.s1*(float)((b0 >> 1) & 1)+ + yl_lo.s2*(float)((b0 >> 2) & 1) + yl_lo.s3*(float)((b0 >> 3) & 1)+ + yl_lo.s4*(float)((b0 >> 4) & 1) + yl_lo.s5*(float)((b0 >> 5) & 1)+ + yl_lo.s6*(float)((b0 >> 6) & 1) + yl_lo.s7*(float)((b0 >> 7) & 1)+ + yl_hi.s0*(float)((b1 >> 0) & 1) + yl_hi.s1*(float)((b1 >> 1) & 1)+ + yl_hi.s2*(float)((b1 >> 2) & 1) + yl_hi.s3*(float)((b1 >> 3) & 1)+ + yl_hi.s4*(float)((b1 >> 4) & 1) + yl_hi.s5*(float)((b1 >> 5) & 1)+ + yl_hi.s6*(float)((b1 >> 6) & 1) + yl_hi.s7*(float)((b1 >> 7) & 1);+ sumf.s0 += (float)ad0[ib] * (2.0f*acc - sumy);++ b0 = ax1[ib*QK1_0_BYTES + il*2 + 0];+ b1 = ax1[ib*QK1_0_BYTES + il*2 + 1];+ acc = yl_lo.s0*(float)((b0 >> 0) & 1) + yl_lo.s1*(float)((b0 >> 1) & 1)+ + yl_lo.s2*(float)((b0 >> 2) & 1) + yl_lo.s3*(float)((b0 >> 3) & 1)+ + yl_lo.s4*(float)((b0 >> 4) & 1) + yl_lo.s5*(float)((b0 >> 5) & 1)+ + yl_lo.s6*(float)((b0 >> 6) & 1) + yl_lo.s7*(float)((b0 >> 7) & 1)+ + yl_hi.s0*(float)((b1 >> 0) & 1) + yl_hi.s1*(float)((b1 >> 1) & 1)+ + yl_hi.s2*(float)((b1 >> 2) & 1) + yl_hi.s3*(float)((b1 >> 3) & 1)+ + yl_hi.s4*(float)((b1 >> 4) & 1) + yl_hi.s5*(float)((b1 >> 5) & 1)+ + yl_hi.s6*(float)((b1 >> 6) & 1) + yl_hi.s7*(float)((b1 >> 7) & 1);+ sumf.s1 += (float)ad1[ib] * (2.0f*acc - sumy);++ b0 = ax2[ib*QK1_0_BYTES + il*2 + 0];+ b1 = ax2[ib*QK1_0_BYTES + il*2 + 1];+ acc = yl_lo.s0*(float)((b0 >> 0) & 1) + yl_lo.s1*(float)((b0 >> 1) & 1)+ + yl_lo.s2*(float)((b0 >> 2) & 1) + yl_lo.s3*(float)((b0 >> 3) & 1)+ + yl_lo.s4*(float)((b0 >> 4) & 1) + yl_lo.s5*(float)((b0 >> 5) & 1)+ + yl_lo.s6*(float)((b0 >> 6) & 1) + yl_lo.s7*(float)((b0 >> 7) & 1)+ + yl_hi.s0*(float)((b1 >> 0) & 1) + yl_hi.s1*(float)((b1 >> 1) & 1)+ + yl_hi.s2*(float)((b1 >> 2) & 1) + yl_hi.s3*(float)((b1 >> 3) & 1)+ + yl_hi.s4*(float)((b1 >> 4) & 1) + yl_hi.s5*(float)((b1 >> 5) & 1)+ + yl_hi.s6*(float)((b1 >> 6) & 1) + yl_hi.s7*(float)((b1 >> 7) & 1);+ sumf.s2 += (float)ad2[ib] * (2.0f*acc - sumy);++ b0 = ax3[ib*QK1_0_BYTES + il*2 + 0];+ b1 = ax3[ib*QK1_0_BYTES + il*2 + 1];+ acc = yl_lo.s0*(float)((b0 >> 0) & 1) + yl_lo.s1*(float)((b0 >> 1) & 1)+ + yl_lo.s2*(float)((b0 >> 2) & 1) + yl_lo.s3*(float)((b0 >> 3) & 1)+ + yl_lo.s4*(float)((b0 >> 4) & 1) + yl_lo.s5*(float)((b0 >> 5) & 1)+ + yl_lo.s6*(float)((b0 >> 6) & 1) + yl_lo.s7*(float)((b0 >> 7) & 1)+ + yl_hi.s0*(float)((b1 >> 0) & 1) + yl_hi.s1*(float)((b1 >> 1) & 1)+ + yl_hi.s2*(float)((b1 >> 2) & 1) + yl_hi.s3*(float)((b1 >> 3) & 1)+ + yl_hi.s4*(float)((b1 >> 4) & 1) + yl_hi.s5*(float)((b1 >> 5) & 1)+ + yl_hi.s6*(float)((b1 >> 6) & 1) + yl_hi.s7*(float)((b1 >> 7) & 1);+ sumf.s3 += (float)ad3[ib] * (2.0f*acc - sumy);++ yb += N_SIMDWIDTH*NB_Q1_0;+ }++ global float * dst_f32 = (global float *) dst + (ulong)im*ne0*ne1 + (ulong)r1*ne0;++ float4 tot = (float4)(+ sub_group_reduce_add(sumf.s0),+ sub_group_reduce_add(sumf.s1),+ sub_group_reduce_add(sumf.s2),+ sub_group_reduce_add(sumf.s3)+ );++ if (get_sub_group_local_id() == 0) {+ if (first_row + 0 < ne01) dst_f32[first_row + 0] = tot.s0;+ if (first_row + 1 < ne01) dst_f32[first_row + 1] = tot.s1;+ if (first_row + 2 < ne01) dst_f32[first_row + 2] = tot.s2;+ if (first_row + 3 < ne01) dst_f32[first_row + 3] = tot.s3;+ }+}