LLM inference in C/C++
C++ ggml
opencl: support CLAMP on non-contiguous views (#30186)
Hongqiang Wanga0e6dc7384ed
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cppindex 52c38c36b0..17ceacc7ae 100644--- a/ggml/src/ggml-opencl/ggml-opencl.cpp+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp@@ -847,7 +847,7 @@ struct ggml_backend_opencl_context {cl_kernel kernel_sigmoid_f32, kernel_sigmoid_f16;cl_kernel kernel_tri;cl_kernel kernel_fill;- cl_kernel kernel_clamp;+ cl_kernel kernel_clamp, kernel_clamp_nc;cl_kernel kernel_geglu, kernel_reglu, kernel_swiglu, kernel_swiglu_oai, kernel_swiglu_clamp, kernel_geglu_erf,kernel_geglu_quick, kernel_geglu_f16, kernel_reglu_f16, kernel_swiglu_f16, kernel_swiglu_clamp_f16,kernel_geglu_erf_f16, kernel_geglu_quick_f16;@@ -1613,6 +1613,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);CL_CHECK((backend_ctx->kernel_clamp = clCreateKernel(backend_ctx->program_clamp, "kernel_clamp", &err), err));+ CL_CHECK((backend_ctx->kernel_clamp_nc = clCreateKernel(backend_ctx->program_clamp, "kernel_clamp_nc", &err), err));GGML_LOG_CONT(".");}@@ -15137,6 +15138,46 @@ static void ggml_cl_clamp(ggml_backend_t backend, const ggml_tensor * src0, consmemcpy(&min, ((int32_t *) dst->op_params) + 0, sizeof(float));memcpy(&max, ((int32_t *) dst->op_params) + 1, sizeof(float));+ if (!ggml_is_contiguous(src0)) {+ const int ne00 = src0->ne[0];+ const int ne01 = src0->ne[1];+ const int ne02 = src0->ne[2];+ const int ne03 = src0->ne[3];+ const cl_ulong nb00 = src0->nb[0];+ const cl_ulong nb01 = src0->nb[1];+ const cl_ulong nb02 = src0->nb[2];+ const cl_ulong nb03 = src0->nb[3];+ const cl_ulong nb0 = dst->nb[0];+ const cl_ulong nb1 = dst->nb[1];+ const cl_ulong nb2 = dst->nb[2];+ const cl_ulong nb3 = dst->nb[3];++ cl_kernel kernel = backend_ctx->kernel_clamp_nc;++ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(float), &min));+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(float), &max));+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00));+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb00));+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01));+ CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02));+ CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03));+ CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb0));+ CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb1));+ CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb2));+ CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb3));++ const int nth = 64;+ size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03};+ size_t local_work_size[] = {(size_t)nth, 1, 1};++ backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);+ return;+ }+cl_kernel kernel = backend_ctx->kernel_clamp;CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));diff --git a/ggml/src/ggml-opencl/kernels/clamp.cl b/ggml/src/ggml-opencl/kernels/clamp.clindex ae6032444e..366a068363 100644--- a/ggml/src/ggml-opencl/kernels/clamp.cl+++ b/ggml/src/ggml-opencl/kernels/clamp.cl@@ -18,3 +18,33 @@ kernel void kernel_clamp(min :(src0[get_global_id(0)] > max ? max : src0[get_global_id(0)]);}++kernel void kernel_clamp_nc(+ global const char * src0,+ ulong offset0,+ global char * dst,+ ulong offsetd,+ float min,+ float max,+ int ne00,+ ulong nb00,+ ulong nb01,+ ulong nb02,+ ulong nb03,+ ulong nb0,+ ulong nb1,+ ulong nb2,+ ulong nb3+) {+ src0 = src0 + offset0;+ dst = dst + offsetd;++ const int i3 = get_group_id(2);+ const int i2 = get_group_id(1);+ const int i1 = get_group_id(0);++ for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) {+ const float x = *(global const float *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);+ *(global float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0) = x < min ? min : (x > max ? max : x);+ }+}