LLM inference in C/C++
C++ ggml
opencl: extend the elementwise and data‐movement op coverage (#27633)
Hongqiang Wang1548a240e360
diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txtindex 8e6e374f36..37e565ef4f 100644--- a/ggml/src/ggml-opencl/CMakeLists.txt+++ b/ggml/src/ggml-opencl/CMakeLists.txt@@ -222,6 +222,7 @@ set(GGML_OPENCL_KERNELSexpexpm1abs+ unary_extsoftpluspadrepeatdiff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cppindex 864da2536c..ad4a995abf 100644--- a/ggml/src/ggml-opencl/ggml-opencl.cpp+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp@@ -826,6 +826,7 @@ struct ggml_backend_opencl_context {cl_kernel kernel_rope_norm_f32, kernel_rope_norm_f16, kernel_rope_neox_f32, kernel_rope_neox_f16;cl_kernel kernel_rope_multi_f32, kernel_rope_multi_f16, kernel_rope_vision_f32, kernel_rope_vision_f16;cl_kernel kernel_cpy_f16_f16, kernel_cpy_f16_f32, kernel_cpy_f32_f16, kernel_cpy_f32_f32, kernel_cpy_f32_f32_pack, kernel_cpy_i32_i32;+ cl_kernel kernel_cpy_f32_f32_flat = nullptr;cl_kernel kernel_mul_mat_f32_f32;cl_kernel kernel_mul_mat_f16_f16;cl_kernel kernel_mul_mat_f16_f32_1row;@@ -932,11 +933,20 @@ struct ggml_backend_opencl_context {cl_kernel kernel_expm1_f16, kernel_expm1_f16_4, kernel_expm1_f16_nc;cl_kernel kernel_abs_f32, kernel_abs_f32_4, kernel_abs_f32_nc;cl_kernel kernel_abs_f16, kernel_abs_f16_4, kernel_abs_f16_nc;+ cl_kernel kernel_sgn_f32, kernel_sgn_f32_4, kernel_sgn_f32_nc, kernel_sgn_f16, kernel_sgn_f16_4, kernel_sgn_f16_nc;+ cl_kernel kernel_step_f32, kernel_step_f32_4, kernel_step_f32_nc, kernel_step_f16, kernel_step_f16_4, kernel_step_f16_nc;+ cl_kernel kernel_elu_f32, kernel_elu_f32_4, kernel_elu_f32_nc, kernel_elu_f16, kernel_elu_f16_4, kernel_elu_f16_nc;+ cl_kernel kernel_hardswish_f32, kernel_hardswish_f32_4, kernel_hardswish_f32_nc, kernel_hardswish_f16, kernel_hardswish_f16_4, kernel_hardswish_f16_nc;+ cl_kernel kernel_hardsigmoid_f32, kernel_hardsigmoid_f32_4, kernel_hardsigmoid_f32_nc, kernel_hardsigmoid_f16, kernel_hardsigmoid_f16_4, kernel_hardsigmoid_f16_nc;+ cl_kernel kernel_floor_f32, kernel_floor_f32_4, kernel_floor_f32_nc, kernel_floor_f16, kernel_floor_f16_4, kernel_floor_f16_nc;+ cl_kernel kernel_ceil_f32, kernel_ceil_f32_4, kernel_ceil_f32_nc, kernel_ceil_f16, kernel_ceil_f16_4, kernel_ceil_f16_nc;+ cl_kernel kernel_round_f32, kernel_round_f32_4, kernel_round_f32_nc, kernel_round_f16, kernel_round_f16_4, kernel_round_f16_nc;+ cl_kernel kernel_trunc_f32, kernel_trunc_f32_4, kernel_trunc_f32_nc, kernel_trunc_f16, kernel_trunc_f16_4, kernel_trunc_f16_nc;cl_kernel kernel_softplus_f32, kernel_softplus_f32_4, kernel_softplus_f32_nc;cl_kernel kernel_softplus_f16, kernel_softplus_f16_4, kernel_softplus_f16_nc;cl_kernel kernel_upscale;cl_kernel kernel_upscale_bilinear;- cl_kernel kernel_concat_f32, kernel_concat_f32_pack;+ cl_kernel kernel_concat_b1, kernel_concat_b2, kernel_concat_b4, kernel_concat_b8, kernel_concat_b4_pack;cl_kernel kernel_conv_2d_f16;cl_kernel kernel_conv_2d_f32;cl_kernel kernel_conv_2d_f16_f32;@@ -1537,6 +1547,13 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {CL_CHECK((backend_ctx->kernel_cpy_f32_f16 = clCreateKernel(prog, "kernel_cpy_f32_f16", &err), err));CL_CHECK((backend_ctx->kernel_cpy_f32_f32 = clCreateKernel(prog, "kernel_cpy_f32_f32", &err), err));CL_CHECK((backend_ctx->kernel_cpy_f32_f32_pack = clCreateKernel(prog, "kernel_cpy_f32_f32_pack", &err), err));+ { // optional: without it ggml_cl_cpy keeps the row-mapped kernel+ cl_int err_flat = CL_SUCCESS;+ cl_kernel k = clCreateKernel(prog, "kernel_cpy_f32_f32_flat", &err_flat);+ if (err_flat == CL_SUCCESS) {+ backend_ctx->kernel_cpy_f32_f32_flat = k;+ }+ }CL_CHECK((backend_ctx->kernel_cpy_i32_i32 = clCreateKernel(prog, "kernel_cpy_i32_i32", &err), err));GGML_LOG_CONT(".");}@@ -3183,6 +3200,38 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {GGML_LOG_CONT(".");}+ // unary_ext (sgn, step, elu, hardswish, hardsigmoid, floor, ceil, round, trunc)+ {+#ifdef GGML_OPENCL_EMBED_KERNELS+ const std::string kernel_src {+ #include "unary_ext.cl.h"+ };+#else+ const std::string kernel_src = read_file("unary_ext.cl");+#endif+ cl_program prog =+ build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);+#define CL_UNARY_EXT_K(op) \+ CL_CHECK((backend_ctx->kernel_##op##_f32 = clCreateKernel(prog, "kernel_" #op "_f32", &err), err)); \+ CL_CHECK((backend_ctx->kernel_##op##_f32_4 = clCreateKernel(prog, "kernel_" #op "_f32_4", &err), err)); \+ CL_CHECK((backend_ctx->kernel_##op##_f32_nc = clCreateKernel(prog, "kernel_" #op "_f32_nc", &err), err)); \+ CL_CHECK((backend_ctx->kernel_##op##_f16 = clCreateKernel(prog, "kernel_" #op "_f16", &err), err)); \+ CL_CHECK((backend_ctx->kernel_##op##_f16_4 = clCreateKernel(prog, "kernel_" #op "_f16_4", &err), err)); \+ CL_CHECK((backend_ctx->kernel_##op##_f16_nc = clCreateKernel(prog, "kernel_" #op "_f16_nc", &err), err));+ CL_UNARY_EXT_K(sgn)+ CL_UNARY_EXT_K(step)+ CL_UNARY_EXT_K(elu)+ CL_UNARY_EXT_K(hardswish)+ CL_UNARY_EXT_K(hardsigmoid)+ CL_UNARY_EXT_K(floor)+ CL_UNARY_EXT_K(ceil)+ CL_UNARY_EXT_K(round)+ CL_UNARY_EXT_K(trunc)+#undef CL_UNARY_EXT_K+ CL_CHECK(clReleaseProgram(prog));+ GGML_LOG_CONT(".");+ }+// softplus{#ifdef GGML_OPENCL_EMBED_KERNELS@@ -3247,8 +3296,11 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {#endifcl_program prog =build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);- CL_CHECK((backend_ctx->kernel_concat_f32 = clCreateKernel(prog, "kernel_concat_f32", &err), err));- CL_CHECK((backend_ctx->kernel_concat_f32_pack = clCreateKernel(prog, "kernel_concat_f32_pack", &err), err));+ CL_CHECK((backend_ctx->kernel_concat_b1 = clCreateKernel(prog, "kernel_concat_b1", &err), err));+ CL_CHECK((backend_ctx->kernel_concat_b2 = clCreateKernel(prog, "kernel_concat_b2", &err), err));+ CL_CHECK((backend_ctx->kernel_concat_b4 = clCreateKernel(prog, "kernel_concat_b4", &err), err));+ CL_CHECK((backend_ctx->kernel_concat_b8 = clCreateKernel(prog, "kernel_concat_b8", &err), err));+ CL_CHECK((backend_ctx->kernel_concat_b4_pack = clCreateKernel(prog, "kernel_concat_b4_pack", &err), err));CL_CHECK(clReleaseProgram(prog));GGML_LOG_CONT(".");}@@ -8452,6 +8504,15 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_tecase GGML_UNARY_OP_EXPM1:return op->src[0]->type == GGML_TYPE_F32;case GGML_UNARY_OP_ABS:+ case GGML_UNARY_OP_SGN:+ case GGML_UNARY_OP_STEP:+ case GGML_UNARY_OP_ELU:+ case GGML_UNARY_OP_HARDSWISH:+ case GGML_UNARY_OP_HARDSIGMOID:+ case GGML_UNARY_OP_FLOOR:+ case GGML_UNARY_OP_CEIL:+ case GGML_UNARY_OP_ROUND:+ case GGML_UNARY_OP_TRUNC:return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16;case GGML_UNARY_OP_SOFTPLUS:return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16;@@ -8531,7 +8592,13 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_tereturn S_v == 16 || S_v == 32 || S_v == 64 || S_v == 128;}case GGML_OP_CONCAT:- return op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32;+ {+ const ggml_type t = op->src[0]->type;+ return op->src[1]->type == t && op->type == t &&+ !ggml_is_quantized(t) && ggml_blck_size(t) == 1 &&+ (ggml_type_size(t) == 1 || ggml_type_size(t) == 2 ||+ ggml_type_size(t) == 4 || ggml_type_size(t) == 8);+ }case GGML_OP_TIMESTEP_EMBEDDING:return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32;case GGML_OP_GROUP_NORM:@@ -15140,6 +15207,97 @@ static void ggml_cl_abs(ggml_backend_t backend, const ggml_tensor * src0, const}}+// Shared driver for the extended unary ops (unary_ext.cl), same selection as+// ggml_cl_abs: contiguous picks the vec4 kernel when the element count is a+// multiple of 4 (else scalar); non-contiguous uses the stride-addressed kernel.+static void ggml_cl_unary_ext(ggml_backend_t backend, const ggml_tensor * src0, ggml_tensor * dst,+ cl_kernel k_f32, cl_kernel k_f32_4, cl_kernel k_f32_nc,+ cl_kernel k_f16, cl_kernel k_f16_4, cl_kernel k_f16_nc) {+ GGML_ASSERT(src0);+ GGML_ASSERT(src0->extra);+ GGML_ASSERT(dst);+ GGML_ASSERT(dst->extra);++ ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;++ ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;+ ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;++ cl_ulong offset0 = extra0->offset + src0->view_offs;+ cl_ulong offsetd = extrad->offset + dst->view_offs;++ const int ne00 = src0->ne[0], ne01 = src0->ne[1], ne02 = src0->ne[2], ne03 = src0->ne[3];+ const cl_ulong nb00 = src0->nb[0], nb01 = src0->nb[1], nb02 = src0->nb[2], nb03 = src0->nb[3];+ const cl_ulong nb0 = dst->nb[0], nb1 = dst->nb[1], nb2 = dst->nb[2], nb3 = dst->nb[3];++ const bool is_f16 = (src0->type == GGML_TYPE_F16);+ cl_kernel kernel;++ if (ggml_is_contiguous(src0)) {+ int n = ggml_nelements(dst);+ if (n % 4 == 0) {+ kernel = is_f16 ? k_f16_4 : k_f32_4;+ n /= 4;+ } else {+ kernel = is_f16 ? k_f16 : k_f32;+ }++ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));++ size_t global_work_size[] = {(size_t)n, 1, 1};+ size_t local_work_size[] = {64, 1, 1};+ size_t * local_work_size_ptr = local_work_size;+ if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) {+ local_work_size_ptr = nullptr;+ }+ backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst);+ } else {+ kernel = is_f16 ? k_f16_nc : k_f32_nc;++ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00));+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00));+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01));+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02));+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03));+ CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0));+ CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1));+ CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2));+ CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3));++ int nth = 64;+ size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03};+ size_t local_work_size[] = {(size_t)nth, 1, 1};+ backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);+ }+}++#define GGML_CL_UNARY_EXT_WRAP(FN, OP) \+static void FN(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { \+ UNUSED(src1); \+ ggml_backend_opencl_context *c = (ggml_backend_opencl_context *)backend->context; \+ ggml_cl_unary_ext(backend, src0, dst, c->kernel_##OP##_f32, c->kernel_##OP##_f32_4, c->kernel_##OP##_f32_nc, \+ c->kernel_##OP##_f16, c->kernel_##OP##_f16_4, c->kernel_##OP##_f16_nc); \+}++GGML_CL_UNARY_EXT_WRAP(ggml_cl_sgn, sgn)+GGML_CL_UNARY_EXT_WRAP(ggml_cl_step, step)+GGML_CL_UNARY_EXT_WRAP(ggml_cl_elu, elu)+GGML_CL_UNARY_EXT_WRAP(ggml_cl_hardswish, hardswish)+GGML_CL_UNARY_EXT_WRAP(ggml_cl_hardsigmoid, hardsigmoid)+GGML_CL_UNARY_EXT_WRAP(ggml_cl_floor, floor)+GGML_CL_UNARY_EXT_WRAP(ggml_cl_ceil, ceil)+GGML_CL_UNARY_EXT_WRAP(ggml_cl_round, round)+GGML_CL_UNARY_EXT_WRAP(ggml_cl_trunc, trunc)++#undef GGML_CL_UNARY_EXT_WRAP+static void ggml_cl_softplus(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {GGML_ASSERT(src0);GGML_ASSERT(src0->extra);@@ -15517,9 +15675,8 @@ static void ggml_cl_concat(ggml_backend_t backend, const ggml_tensor * src0, conGGML_ASSERT(src1->extra);GGML_ASSERT(dst);GGML_ASSERT(dst->extra);- GGML_ASSERT(src0->type == GGML_TYPE_F32);- GGML_ASSERT(src1->type == GGML_TYPE_F32);- GGML_ASSERT(dst->type == GGML_TYPE_F32);+ GGML_ASSERT(src0->type == src1->type);+ GGML_ASSERT(src0->type == dst->type);ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;@@ -15561,9 +15718,21 @@ static void ggml_cl_concat(ggml_backend_t backend, const ggml_tensor * src0, conint nth = MIN(64, ne0);- const bool concat_pack = (dim == 0 && ne0 < 32);- cl_kernel kernel = concat_pack ? backend_ctx->kernel_concat_f32_pack- : backend_ctx->kernel_concat_f32;+ const size_t ts = ggml_type_size(dst->type);+ // the pack kernel copies 4-byte elements, so it is only valid for those.+ const bool concat_pack = (dim == 0 && ne0 < 32 && ts == 4);+ cl_kernel kernel;+ if (concat_pack) {+ kernel = backend_ctx->kernel_concat_b4_pack;+ } else {+ switch (ts) {+ case 1: kernel = backend_ctx->kernel_concat_b1; break;+ case 2: kernel = backend_ctx->kernel_concat_b2; break;+ case 4: kernel = backend_ctx->kernel_concat_b4; break;+ case 8: kernel = backend_ctx->kernel_concat_b8; break;+ default: GGML_ABORT("unsupported concat element size: %zu", ts);+ }+ }CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));@@ -26021,6 +26190,38 @@ static void ggml_cl_cpy(ggml_backend_t backend, const ggml_tensor * src0, constcl_ulong offset0 = extra0->offset + src0->view_offs;cl_ulong offset1 = extra1->offset + src1->view_offs;+ // A contiguous f32 -> f32 copy is a linear move. The kernel below maps one workgroup to+ // each row, so a tensor with few long rows runs on a single compute unit; dispatch those+ // over the whole device instead. GGML_OPENCL_CPY_FLAT=0 restores the row-mapped path.+ static const bool cpy_flat_on = []{+ const char * e = getenv("GGML_OPENCL_CPY_FLAT");+ return !(e && e[0] == '0');+ }();+ if (cpy_flat_on && backend_ctx->kernel_cpy_f32_f32_flat != nullptr &&+ src0t == GGML_TYPE_F32 && src1t == GGML_TYPE_F32 &&+ ggml_is_contiguous(src0) && ggml_is_contiguous(src1) &&+ ggml_nelements(src0) == ggml_nelements(src1)) {+ cl_kernel k = backend_ctx->kernel_cpy_f32_f32_flat;+ const cl_ulong nelem = (cl_ulong) ggml_nelements(src0);+ const cl_ulong n4 = nelem / 4;++ CL_CHECK(clSetKernelArg(k, 0, sizeof(cl_mem), &extra0->data_device));+ CL_CHECK(clSetKernelArg(k, 1, sizeof(cl_ulong), &offset0));+ CL_CHECK(clSetKernelArg(k, 2, sizeof(cl_mem), &extra1->data_device));+ CL_CHECK(clSetKernelArg(k, 3, sizeof(cl_ulong), &offset1));+ CL_CHECK(clSetKernelArg(k, 4, sizeof(cl_ulong), &nelem));+ CL_CHECK(clSetKernelArg(k, 5, sizeof(cl_ulong), &n4));++ // one work item per float4, plus one for the trailing scalars+ const size_t items = (size_t) n4 + ((nelem % 4) ? 1 : 0);+ const size_t lsz = MIN((size_t) 64, backend_ctx->max_workgroup_size);+ size_t global_work_size[] = { ((items + lsz - 1) / lsz) * lsz, 1, 1 };+ size_t local_work_size[] = { lsz, 1, 1 };++ backend_ctx->enqueue_ndrange_kernel(k, 1, global_work_size, local_work_size, src1);+ return;+ }+cl_kernel kernel;switch (src0t) {@@ -27454,6 +27655,42 @@ bool ggml_cl_compute_forward(ggml_backend_t backend, struct ggml_tensor * tensor}func = ggml_cl_abs;break;+ case GGML_UNARY_OP_SGN:+ if (!any_on_device) { return false; }+ func = ggml_cl_sgn;+ break;+ case GGML_UNARY_OP_STEP:+ if (!any_on_device) { return false; }+ func = ggml_cl_step;+ break;+ case GGML_UNARY_OP_ELU:+ if (!any_on_device) { return false; }+ func = ggml_cl_elu;+ break;+ case GGML_UNARY_OP_HARDSWISH:+ if (!any_on_device) { return false; }+ func = ggml_cl_hardswish;+ break;+ case GGML_UNARY_OP_HARDSIGMOID:+ if (!any_on_device) { return false; }+ func = ggml_cl_hardsigmoid;+ break;+ case GGML_UNARY_OP_FLOOR:+ if (!any_on_device) { return false; }+ func = ggml_cl_floor;+ break;+ case GGML_UNARY_OP_CEIL:+ if (!any_on_device) { return false; }+ func = ggml_cl_ceil;+ break;+ case GGML_UNARY_OP_ROUND:+ if (!any_on_device) { return false; }+ func = ggml_cl_round;+ break;+ case GGML_UNARY_OP_TRUNC:+ if (!any_on_device) { return false; }+ func = ggml_cl_trunc;+ break;case GGML_UNARY_OP_SOFTPLUS:if (!any_on_device) {return false;diff --git a/ggml/src/ggml-opencl/kernels/concat.cl b/ggml/src/ggml-opencl/kernels/concat.clindex 2fbd7851d3..8ecf7466d6 100644--- a/ggml/src/ggml-opencl/kernels/concat.cl+++ b/ggml/src/ggml-opencl/kernels/concat.cl@@ -1,56 +1,66 @@-kernel void kernel_concat_f32(- global const char * src0,- ulong offset0,- global const char * src1,- ulong offset1,- global char * dst,- ulong offsetd,- int ne00,- int ne01,- int ne02,- int ne03,- ulong nb00,- ulong nb01,- ulong nb02,- ulong nb03,- ulong nb10,- ulong nb11,- ulong nb12,- ulong nb13,- int ne0,- ulong nb0,- ulong nb1,- ulong nb2,- ulong nb3,- int dim-) {- src0 = src0 + offset0;- src1 = src1 + offset1;- dst = dst + offsetd;-- const int i3 = get_group_id(2);- const int i2 = get_group_id(1);- const int i1 = get_group_id(0);-- int o[4] = {0, 0, 0, 0};- o[dim] = dim == 0 ? ne00 : (dim == 1 ? ne01 : (dim == 2 ? ne02 : ne03));-- global const float * x;-- for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {- if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {- x = (global const float *)(src0 + (i3 )*nb03 + (i2 )*nb02 + (i1 )*nb01 + (i0 )*nb00);- } else {- x = (global const float *)(src1 + (i3 - o[3])*nb13 + (i2 - o[2])*nb12 + (i1 - o[1])*nb11 + (i0 - o[0])*nb10);- }-- global float * y = (global float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);+// concat is a pure copy, so the kernels are keyed by element byte size+// (1/2/4/8) rather than logical type, matching the CUDA backend.- *y = *x;- }+#define KERNEL_CONCAT(SUFFIX, T) \+kernel void kernel_concat_##SUFFIX( \+ global const char * src0, \+ ulong offset0, \+ global const char * src1, \+ ulong offset1, \+ global char * dst, \+ ulong offsetd, \+ int ne00, \+ int ne01, \+ int ne02, \+ int ne03, \+ ulong nb00, \+ ulong nb01, \+ ulong nb02, \+ ulong nb03, \+ ulong nb10, \+ ulong nb11, \+ ulong nb12, \+ ulong nb13, \+ int ne0, \+ ulong nb0, \+ ulong nb1, \+ ulong nb2, \+ ulong nb3, \+ int dim \+) { \+ src0 = src0 + offset0; \+ src1 = src1 + offset1; \+ dst = dst + offsetd; \+ \+ const int i3 = get_group_id(2); \+ const int i2 = get_group_id(1); \+ const int i1 = get_group_id(0); \+ \+ int o[4] = {0, 0, 0, 0}; \+ o[dim] = dim == 0 ? ne00 : (dim == 1 ? ne01 : (dim == 2 ? ne02 : ne03)); \+ \+ global const T * x; \+ \+ for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) { \+ if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) { \+ x = (global const T *)(src0 + (i3 )*nb03 + (i2 )*nb02 + (i1 )*nb01 + (i0 )*nb00); \+ } else { \+ x = (global const T *)(src1 + (i3 - o[3])*nb13 + (i2 - o[2])*nb12 + (i1 - o[1])*nb11 + (i0 - o[0])*nb10); \+ } \+ \+ global T * y = (global T *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0); \+ \+ *y = *x; \+ } \}-kernel void kernel_concat_f32_pack(+KERNEL_CONCAT(b1, char)+KERNEL_CONCAT(b2, short)+KERNEL_CONCAT(b4, int)+KERNEL_CONCAT(b8, long)++// packed variant for the common dim==0, small-ne0 case (4-byte elements only).+kernel void kernel_concat_b4_pack(global const char * src0,ulong offset0,global const char * src1,@@ -104,14 +114,14 @@ kernel void kernel_concat_f32_pack(o[dim] = dim == 0 ? ne00 : (dim == 1 ? ne01 : (dim == 2 ? ne02 : ne03));for (int i0 = lane; i0 < ne0; i0 += tpr) {- global const float * x;+ global const int * x;if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {- x = (global const float *)(src0 + (i3 )*nb03 + (i2 )*nb02 + (i1 )*nb01 + (i0 )*nb00);+ x = (global const int *)(src0 + (i3 )*nb03 + (i2 )*nb02 + (i1 )*nb01 + (i0 )*nb00);} else {- x = (global const float *)(src1 + (i3 - o[3])*nb13 + (i2 - o[2])*nb12 + (i1 - o[1])*nb11 + (i0 - o[0])*nb10);+ x = (global const int *)(src1 + (i3 - o[3])*nb13 + (i2 - o[2])*nb12 + (i1 - o[1])*nb11 + (i0 - o[0])*nb10);}- global float * y = (global float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);+ global int * y = (global int *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);*y = *x;}diff --git a/ggml/src/ggml-opencl/kernels/cpy.cl b/ggml/src/ggml-opencl/kernels/cpy.clindex adbd2e766d..e875bfaf75 100644--- a/ggml/src/ggml-opencl/kernels/cpy.cl+++ b/ggml/src/ggml-opencl/kernels/cpy.cl@@ -286,3 +286,28 @@ kernel void kernel_cpy_i32_i32(dst_data[i00] = src[0];}}++// Contiguous f32 copy, one work item per float4 over the whole tensor. The kernels above map+// one workgroup to each row, which leaves a tensor with few long rows on a single compute unit.+// vload4/vstore4 rather than a float4 cast: these buffers carry an arbitrary 4-byte view offset.+kernel void kernel_cpy_f32_f32_flat(+ global float * src0,+ ulong offset0,+ global float * dst,+ ulong offsetd,+ ulong ne,+ ulong n4+) {+ src0 = (global float*)((global char*)src0 + offset0);+ dst = (global float*)((global char*)dst + offsetd);++ const ulong i = get_global_id(0);++ if (i < n4) {+ vstore4(vload4(i, src0), i, dst);+ } else if (i == n4) {+ for (ulong t = n4 * 4; t < ne; ++t) {+ dst[t] = src0[t];+ }+ }+}diff --git a/ggml/src/ggml-opencl/kernels/unary_ext.cl b/ggml/src/ggml-opencl/kernels/unary_ext.clnew file mode 100644index 0000000000..e86eadfa5f--- /dev/null+++ b/ggml/src/ggml-opencl/kernels/unary_ext.cl@@ -0,0 +1,85 @@+#pragma OPENCL EXTENSION cl_khr_fp16 : enable++//------------------------------------------------------------------------------+// Extended elementwise unary ops, same variant shape as abs.cl:+// f32, f32_4 (vec4), f16, f16_4 (vec4), f32_nc, f16_nc (stride-addressed).+//+// sgn, step, elu, hardswish, hardsigmoid, floor, ceil, round, trunc.+//+// Semantics match the ggml CPU reference (ggml.c). Values are computed in float+// (the f16 variants read/write half and convert), so the conditional ops match+// the CPU bit-for-bit within tolerance. SEXPR is the scalar form, VEXPR the+// float4 form (vector ternaries need select()).+//------------------------------------------------------------------------------++#define UNARY_EXT(NAME, SEXPR, VEXPR) \+kernel void kernel_##NAME##_f32( \+ global const float * src0, ulong offset0, \+ global float * dst, ulong offsetd) { \+ src0 = (global float*)((global char*)src0 + offset0); \+ dst = (global float*)((global char*)dst + offsetd); \+ float x = src0[get_global_id(0)]; \+ dst[get_global_id(0)] = (SEXPR); \+} \+kernel void kernel_##NAME##_f32_4( \+ global const float4 * src0, ulong offset0, \+ global float4 * dst, ulong offsetd) { \+ src0 = (global float4*)((global char*)src0 + offset0); \+ dst = (global float4*)((global char*)dst + offsetd); \+ float4 x = src0[get_global_id(0)]; \+ dst[get_global_id(0)] = (VEXPR); \+} \+kernel void kernel_##NAME##_f16( \+ global const half * src0, ulong offset0, \+ global half * dst, ulong offsetd) { \+ src0 = (global half*)((global char*)src0 + offset0); \+ dst = (global half*)((global char*)dst + offsetd); \+ float x = src0[get_global_id(0)]; \+ dst[get_global_id(0)] = (SEXPR); \+} \+kernel void kernel_##NAME##_f16_4( \+ global const half4 * src0, ulong offset0, \+ global half4 * dst, ulong offsetd) { \+ src0 = (global half4*)((global char*)src0 + offset0); \+ dst = (global half4*)((global char*)dst + offsetd); \+ float4 x = convert_float4(src0[get_global_id(0)]); \+ dst[get_global_id(0)] = convert_half4(VEXPR); \+} \+kernel void kernel_##NAME##_f32_nc( \+ global const char * src0, ulong offset0, \+ global char * dst, ulong offsetd, \+ int ne00, ulong nb00, ulong nb01, ulong nb02, ulong nb03, \+ ulong nb0, ulong nb1, ulong nb2, ulong nb3) { \+ src0 = src0 + offset0; dst = dst + offsetd; \+ const int i3 = get_group_id(2); \+ const int i2 = get_group_id(1); \+ const int i1 = get_group_id(0); \+ for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) { \+ float x = *(global const float *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00); \+ *(global float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0) = (SEXPR); \+ } \+} \+kernel void kernel_##NAME##_f16_nc( \+ global const char * src0, ulong offset0, \+ global char * dst, ulong offsetd, \+ int ne00, ulong nb00, ulong nb01, ulong nb02, ulong nb03, \+ ulong nb0, ulong nb1, ulong nb2, ulong nb3) { \+ src0 = src0 + offset0; dst = dst + offsetd; \+ const int i3 = get_group_id(2); \+ const int i2 = get_group_id(1); \+ const int i1 = get_group_id(0); \+ for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) {\+ float x = *(global const half *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00); \+ *(global half *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0) = (SEXPR); \+ } \+}++UNARY_EXT(sgn, sign(x), sign(x))+UNARY_EXT(step, x > 0.0f ? 1.0f : 0.0f, select((float4)0.0f, (float4)1.0f, x > 0.0f))+UNARY_EXT(elu, x > 0.0f ? x : expm1(x), select(expm1(x), x, x > 0.0f))+UNARY_EXT(hardswish, x * fmin(1.0f, fmax(0.0f, (x + 3.0f) / 6.0f)), x * fmin((float4)1.0f, fmax((float4)0.0f, (x + 3.0f) / 6.0f)))+UNARY_EXT(hardsigmoid, fmin(1.0f, fmax(0.0f, (x + 3.0f) / 6.0f)), fmin((float4)1.0f, fmax((float4)0.0f, (x + 3.0f) / 6.0f)))+UNARY_EXT(floor, floor(x), floor(x))+UNARY_EXT(ceil, ceil(x), ceil(x))+UNARY_EXT(round, round(x), round(x))+UNARY_EXT(trunc, trunc(x), trunc(x))