LLM inference in C/C++
C++ ggml
opencl: add ABS op (#25115)
Gezahegneb85833e93412
diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txtindex ff3ad7e34d..1a534f884f 100644--- a/ggml/src/ggml-opencl/CMakeLists.txt+++ b/ggml/src/ggml-opencl/CMakeLists.txt@@ -211,6 +211,7 @@ set(GGML_OPENCL_KERNELStanhexpexpm1+ abssoftpluspadrepeatdiff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cppindex b8e7a5614e..61d2fbed26 100644--- a/ggml/src/ggml-opencl/ggml-opencl.cpp+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp@@ -845,6 +845,8 @@ struct ggml_backend_opencl_context {cl_kernel kernel_exp_f16, kernel_exp_f16_4, kernel_exp_f16_nc;cl_kernel kernel_expm1_f32, kernel_expm1_f32_4, kernel_expm1_f32_nc;cl_kernel kernel_expm1_f16, kernel_expm1_f16_4, kernel_expm1_f16_nc;+ cl_kernel kernel_abs_f32, kernel_abs_f32_4, kernel_abs_f32_nc;+ cl_kernel kernel_abs_f16, kernel_abs_f16_4, kernel_abs_f16_nc;cl_kernel kernel_softplus_f32, kernel_softplus_f32_4, kernel_softplus_f32_nc;cl_kernel kernel_softplus_f16, kernel_softplus_f16_4, kernel_softplus_f16_nc;cl_kernel kernel_upscale;@@ -2929,6 +2931,27 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {GGML_LOG_CONT(".");}+ // abs+ {+#ifdef GGML_OPENCL_EMBED_KERNELS+ const std::string kernel_src {+ #include "abs.cl.h"+ };+#else+ const std::string kernel_src = read_file("abs.cl");+#endif+ cl_program prog =+ build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts);+ CL_CHECK((backend_ctx->kernel_abs_f32 = clCreateKernel(prog, "kernel_abs_f32", &err), err));+ CL_CHECK((backend_ctx->kernel_abs_f32_4 = clCreateKernel(prog, "kernel_abs_f32_4", &err), err));+ CL_CHECK((backend_ctx->kernel_abs_f32_nc = clCreateKernel(prog, "kernel_abs_f32_nc", &err), err));+ CL_CHECK((backend_ctx->kernel_abs_f16 = clCreateKernel(prog, "kernel_abs_f16", &err), err));+ CL_CHECK((backend_ctx->kernel_abs_f16_4 = clCreateKernel(prog, "kernel_abs_f16_4", &err), err));+ CL_CHECK((backend_ctx->kernel_abs_f16_nc = clCreateKernel(prog, "kernel_abs_f16_nc", &err), err));+ CL_CHECK(clReleaseProgram(prog));+ GGML_LOG_CONT(".");+ }+// softplus{#ifdef GGML_OPENCL_EMBED_KERNELS@@ -7153,6 +7176,8 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_tereturn op->src[0]->type == GGML_TYPE_F32;case GGML_UNARY_OP_EXPM1:return op->src[0]->type == GGML_TYPE_F32;+ case GGML_UNARY_OP_ABS:+ return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16;case GGML_UNARY_OP_SOFTPLUS:return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16;default:@@ -13387,6 +13412,102 @@ static void ggml_cl_expm1(ggml_backend_t backend, const ggml_tensor * src0, cons}}+static void ggml_cl_abs(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {+ GGML_ASSERT(src0);+ GGML_ASSERT(src0->extra);+ GGML_ASSERT(dst);+ GGML_ASSERT(dst->extra);++ UNUSED(src1);++ ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;++ ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;+ ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;++ cl_ulong offset0 = extra0->offset + src0->view_offs;+ cl_ulong offsetd = extrad->offset + dst->view_offs;++ const int ne00 = src0->ne[0];+ const int ne01 = src0->ne[1];+ const int ne02 = src0->ne[2];+ const int ne03 = src0->ne[3];++ const cl_ulong nb00 = src0->nb[0];+ const cl_ulong nb01 = src0->nb[1];+ const cl_ulong nb02 = src0->nb[2];+ const cl_ulong nb03 = src0->nb[3];++ const cl_ulong nb0 = dst->nb[0];+ const cl_ulong nb1 = dst->nb[1];+ const cl_ulong nb2 = dst->nb[2];+ const cl_ulong nb3 = dst->nb[3];++ cl_kernel kernel;++ if (ggml_is_contiguous(src0)) {+ // Handle contiguous input+ int n = ggml_nelements(dst);+ if (n % 4 == 0) {+ if (src0->type == GGML_TYPE_F32) {+ kernel = backend_ctx->kernel_abs_f32_4;+ } else {+ kernel = backend_ctx->kernel_abs_f16_4;+ }+ n /= 4;+ } else {+ if (src0->type == GGML_TYPE_F32) {+ kernel = backend_ctx->kernel_abs_f32;+ } else {+ kernel = backend_ctx->kernel_abs_f16;+ }+ }++ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));++ size_t global_work_size[] = {(size_t)n, 1, 1};+ size_t local_work_size[] = {64, 1, 1};++ size_t * local_work_size_ptr = local_work_size;+ if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) {+ local_work_size_ptr = nullptr;+ }++ backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst);+ } else {+ // Handle non-contiguous input+ if (src0->type == GGML_TYPE_F32) {+ kernel = backend_ctx->kernel_abs_f32_nc;+ } else {+ kernel = backend_ctx->kernel_abs_f16_nc;+ }++ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00));+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00));+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01));+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02));+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03));+ CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0));+ CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1));+ CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2));+ CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3));++ int nth = 64;++ size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03};+ size_t local_work_size[] = {(size_t)nth, 1, 1};++ backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);+ }+}+static void ggml_cl_softplus(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {GGML_ASSERT(src0);GGML_ASSERT(src0->extra);@@ -24408,6 +24529,12 @@ bool ggml_cl_compute_forward(ggml_backend_t backend, struct ggml_tensor * tensor}func = ggml_cl_expm1;break;+ case GGML_UNARY_OP_ABS:+ if (!any_on_device) {+ return false;+ }+ func = ggml_cl_abs;+ break;case GGML_UNARY_OP_SOFTPLUS:if (!any_on_device) {return false;diff --git a/ggml/src/ggml-opencl/kernels/abs.cl b/ggml/src/ggml-opencl/kernels/abs.clnew file mode 100644index 0000000000..96e952c284--- /dev/null+++ b/ggml/src/ggml-opencl/kernels/abs.cl@@ -0,0 +1,113 @@+#pragma OPENCL EXTENSION cl_khr_fp16 : enable++//------------------------------------------------------------------------------+// abs+//------------------------------------------------------------------------------++kernel void kernel_abs_f32(+ global const float * src0,+ ulong offset0,+ global float * dst,+ ulong offsetd+) {+ src0 = (global float*)((global char*)src0 + offset0);+ dst = (global float*)((global char*)dst + offsetd);++ dst[get_global_id(0)] = fabs(src0[get_global_id(0)]);+}++kernel void kernel_abs_f32_4(+ global const float4 * src0,+ ulong offset0,+ global float4 * dst,+ ulong offsetd+) {+ src0 = (global float4*)((global char*)src0 + offset0);+ dst = (global float4*)((global char*)dst + offsetd);++ dst[get_global_id(0)] = fabs(src0[get_global_id(0)]);+}++kernel void kernel_abs_f16(+ global const half * src0,+ ulong offset0,+ global half * dst,+ ulong offsetd+) {+ src0 = (global half*)((global char*)src0 + offset0);+ dst = (global half*)((global char*)dst + offsetd);++ dst[get_global_id(0)] = fabs(src0[get_global_id(0)]);+}++kernel void kernel_abs_f16_4(+ global const half4 * src0,+ ulong offset0,+ global half4 * dst,+ ulong offsetd+) {+ src0 = (global half4*)((global char*)src0 + offset0);+ dst = (global half4*)((global char*)dst + offsetd);++ dst[get_global_id(0)] = fabs(src0[get_global_id(0)]);+}++kernel void kernel_abs_f32_nc(+ global const char * src0,+ ulong offset0,+ global char * dst,+ ulong offsetd,+ int ne00,+ ulong nb00,+ ulong nb01,+ ulong nb02,+ ulong nb03,+ ulong nb0,+ ulong nb1,+ ulong nb2,+ ulong nb3+) {+ src0 = src0 + offset0;+ dst = dst + offsetd;++ const int i3 = get_group_id(2);+ const int i2 = get_group_id(1);+ const int i1 = get_group_id(0);++ for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) {+ global const float * x = (global const float *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);+ global float * y = (global float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);++ *y = fabs(*x);+ }+}++kernel void kernel_abs_f16_nc(+ global const char * src0,+ ulong offset0,+ global char * dst,+ ulong offsetd,+ int ne00,+ ulong nb00,+ ulong nb01,+ ulong nb02,+ ulong nb03,+ ulong nb0,+ ulong nb1,+ ulong nb2,+ ulong nb3+) {+ src0 = src0 + offset0;+ dst = dst + offsetd;++ const int i3 = get_group_id(2);+ const int i2 = get_group_id(1);+ const int i1 = get_group_id(0);++ for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) {+ global const half * x = (global const half *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);+ global half * y = (global half *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);++ *y = fabs(*x);+ }+}