Skip to content

LLM inference in C/C++

C++ ggml

opencl: extend the elementwise and data‐movement op coverage (#27633)

Hongqiang Wang1548a240e360
diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txt
index 8e6e374f36..37e565ef4f 100644
--- a/ggml/src/ggml-opencl/CMakeLists.txt
+++ b/ggml/src/ggml-opencl/CMakeLists.txt
@@ -222,6 +222,7 @@ set(GGML_OPENCL_KERNELS
exp
expm1
abs
+ unary_ext
softplus
pad
repeat
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp
index 864da2536c..ad4a995abf 100644
--- a/ggml/src/ggml-opencl/ggml-opencl.cpp
+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp
@@ -826,6 +826,7 @@ struct ggml_backend_opencl_context {
cl_kernel kernel_rope_norm_f32, kernel_rope_norm_f16, kernel_rope_neox_f32, kernel_rope_neox_f16;
cl_kernel kernel_rope_multi_f32, kernel_rope_multi_f16, kernel_rope_vision_f32, kernel_rope_vision_f16;
cl_kernel kernel_cpy_f16_f16, kernel_cpy_f16_f32, kernel_cpy_f32_f16, kernel_cpy_f32_f32, kernel_cpy_f32_f32_pack, kernel_cpy_i32_i32;
+ cl_kernel kernel_cpy_f32_f32_flat = nullptr;
cl_kernel kernel_mul_mat_f32_f32;
cl_kernel kernel_mul_mat_f16_f16;
cl_kernel kernel_mul_mat_f16_f32_1row;
@@ -932,11 +933,20 @@ struct ggml_backend_opencl_context {
cl_kernel kernel_expm1_f16, kernel_expm1_f16_4, kernel_expm1_f16_nc;
cl_kernel kernel_abs_f32, kernel_abs_f32_4, kernel_abs_f32_nc;
cl_kernel kernel_abs_f16, kernel_abs_f16_4, kernel_abs_f16_nc;
+ cl_kernel kernel_sgn_f32, kernel_sgn_f32_4, kernel_sgn_f32_nc, kernel_sgn_f16, kernel_sgn_f16_4, kernel_sgn_f16_nc;
+ cl_kernel kernel_step_f32, kernel_step_f32_4, kernel_step_f32_nc, kernel_step_f16, kernel_step_f16_4, kernel_step_f16_nc;
+ cl_kernel kernel_elu_f32, kernel_elu_f32_4, kernel_elu_f32_nc, kernel_elu_f16, kernel_elu_f16_4, kernel_elu_f16_nc;
+ cl_kernel kernel_hardswish_f32, kernel_hardswish_f32_4, kernel_hardswish_f32_nc, kernel_hardswish_f16, kernel_hardswish_f16_4, kernel_hardswish_f16_nc;
+ cl_kernel kernel_hardsigmoid_f32, kernel_hardsigmoid_f32_4, kernel_hardsigmoid_f32_nc, kernel_hardsigmoid_f16, kernel_hardsigmoid_f16_4, kernel_hardsigmoid_f16_nc;
+ cl_kernel kernel_floor_f32, kernel_floor_f32_4, kernel_floor_f32_nc, kernel_floor_f16, kernel_floor_f16_4, kernel_floor_f16_nc;
+ cl_kernel kernel_ceil_f32, kernel_ceil_f32_4, kernel_ceil_f32_nc, kernel_ceil_f16, kernel_ceil_f16_4, kernel_ceil_f16_nc;
+ cl_kernel kernel_round_f32, kernel_round_f32_4, kernel_round_f32_nc, kernel_round_f16, kernel_round_f16_4, kernel_round_f16_nc;
+ cl_kernel kernel_trunc_f32, kernel_trunc_f32_4, kernel_trunc_f32_nc, kernel_trunc_f16, kernel_trunc_f16_4, kernel_trunc_f16_nc;
cl_kernel kernel_softplus_f32, kernel_softplus_f32_4, kernel_softplus_f32_nc;
cl_kernel kernel_softplus_f16, kernel_softplus_f16_4, kernel_softplus_f16_nc;
cl_kernel kernel_upscale;
cl_kernel kernel_upscale_bilinear;
- cl_kernel kernel_concat_f32, kernel_concat_f32_pack;
+ cl_kernel kernel_concat_b1, kernel_concat_b2, kernel_concat_b4, kernel_concat_b8, kernel_concat_b4_pack;
cl_kernel kernel_conv_2d_f16;
cl_kernel kernel_conv_2d_f32;
cl_kernel kernel_conv_2d_f16_f32;
@@ -1537,6 +1547,13 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
CL_CHECK((backend_ctx->kernel_cpy_f32_f16 = clCreateKernel(prog, "kernel_cpy_f32_f16", &err), err));
CL_CHECK((backend_ctx->kernel_cpy_f32_f32 = clCreateKernel(prog, "kernel_cpy_f32_f32", &err), err));
CL_CHECK((backend_ctx->kernel_cpy_f32_f32_pack = clCreateKernel(prog, "kernel_cpy_f32_f32_pack", &err), err));
+ { // optional: without it ggml_cl_cpy keeps the row-mapped kernel
+ cl_int err_flat = CL_SUCCESS;
+ cl_kernel k = clCreateKernel(prog, "kernel_cpy_f32_f32_flat", &err_flat);
+ if (err_flat == CL_SUCCESS) {
+ backend_ctx->kernel_cpy_f32_f32_flat = k;
+ }
+ }
CL_CHECK((backend_ctx->kernel_cpy_i32_i32 = clCreateKernel(prog, "kernel_cpy_i32_i32", &err), err));
GGML_LOG_CONT(".");
}
@@ -3183,6 +3200,38 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
GGML_LOG_CONT(".");
}
+ // unary_ext (sgn, step, elu, hardswish, hardsigmoid, floor, ceil, round, trunc)
+ {
+#ifdef GGML_OPENCL_EMBED_KERNELS
+ const std::string kernel_src {
+ #include "unary_ext.cl.h"
+ };
+#else
+ const std::string kernel_src = read_file("unary_ext.cl");
+#endif
+ cl_program prog =
+ build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
+#define CL_UNARY_EXT_K(op) \
+ CL_CHECK((backend_ctx->kernel_##op##_f32 = clCreateKernel(prog, "kernel_" #op "_f32", &err), err)); \
+ CL_CHECK((backend_ctx->kernel_##op##_f32_4 = clCreateKernel(prog, "kernel_" #op "_f32_4", &err), err)); \
+ CL_CHECK((backend_ctx->kernel_##op##_f32_nc = clCreateKernel(prog, "kernel_" #op "_f32_nc", &err), err)); \
+ CL_CHECK((backend_ctx->kernel_##op##_f16 = clCreateKernel(prog, "kernel_" #op "_f16", &err), err)); \
+ CL_CHECK((backend_ctx->kernel_##op##_f16_4 = clCreateKernel(prog, "kernel_" #op "_f16_4", &err), err)); \
+ CL_CHECK((backend_ctx->kernel_##op##_f16_nc = clCreateKernel(prog, "kernel_" #op "_f16_nc", &err), err));
+ CL_UNARY_EXT_K(sgn)
+ CL_UNARY_EXT_K(step)
+ CL_UNARY_EXT_K(elu)
+ CL_UNARY_EXT_K(hardswish)
+ CL_UNARY_EXT_K(hardsigmoid)
+ CL_UNARY_EXT_K(floor)
+ CL_UNARY_EXT_K(ceil)
+ CL_UNARY_EXT_K(round)
+ CL_UNARY_EXT_K(trunc)
+#undef CL_UNARY_EXT_K
+ CL_CHECK(clReleaseProgram(prog));
+ GGML_LOG_CONT(".");
+ }
+
// softplus
{
#ifdef GGML_OPENCL_EMBED_KERNELS
@@ -3247,8 +3296,11 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
#endif
cl_program prog =
build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);
- CL_CHECK((backend_ctx->kernel_concat_f32 = clCreateKernel(prog, "kernel_concat_f32", &err), err));
- CL_CHECK((backend_ctx->kernel_concat_f32_pack = clCreateKernel(prog, "kernel_concat_f32_pack", &err), err));
+ CL_CHECK((backend_ctx->kernel_concat_b1 = clCreateKernel(prog, "kernel_concat_b1", &err), err));
+ CL_CHECK((backend_ctx->kernel_concat_b2 = clCreateKernel(prog, "kernel_concat_b2", &err), err));
+ CL_CHECK((backend_ctx->kernel_concat_b4 = clCreateKernel(prog, "kernel_concat_b4", &err), err));
+ CL_CHECK((backend_ctx->kernel_concat_b8 = clCreateKernel(prog, "kernel_concat_b8", &err), err));
+ CL_CHECK((backend_ctx->kernel_concat_b4_pack = clCreateKernel(prog, "kernel_concat_b4_pack", &err), err));
CL_CHECK(clReleaseProgram(prog));
GGML_LOG_CONT(".");
}
@@ -8452,6 +8504,15 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
case GGML_UNARY_OP_EXPM1:
return op->src[0]->type == GGML_TYPE_F32;
case GGML_UNARY_OP_ABS:
+ case GGML_UNARY_OP_SGN:
+ case GGML_UNARY_OP_STEP:
+ case GGML_UNARY_OP_ELU:
+ case GGML_UNARY_OP_HARDSWISH:
+ case GGML_UNARY_OP_HARDSIGMOID:
+ case GGML_UNARY_OP_FLOOR:
+ case GGML_UNARY_OP_CEIL:
+ case GGML_UNARY_OP_ROUND:
+ case GGML_UNARY_OP_TRUNC:
return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16;
case GGML_UNARY_OP_SOFTPLUS:
return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16;
@@ -8531,7 +8592,13 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
return S_v == 16 || S_v == 32 || S_v == 64 || S_v == 128;
}
case GGML_OP_CONCAT:
- return op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32;
+ {
+ const ggml_type t = op->src[0]->type;
+ return op->src[1]->type == t && op->type == t &&
+ !ggml_is_quantized(t) && ggml_blck_size(t) == 1 &&
+ (ggml_type_size(t) == 1 || ggml_type_size(t) == 2 ||
+ ggml_type_size(t) == 4 || ggml_type_size(t) == 8);
+ }
case GGML_OP_TIMESTEP_EMBEDDING:
return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32;
case GGML_OP_GROUP_NORM:
@@ -15140,6 +15207,97 @@ static void ggml_cl_abs(ggml_backend_t backend, const ggml_tensor * src0, const
}
}
+// Shared driver for the extended unary ops (unary_ext.cl), same selection as
+// ggml_cl_abs: contiguous picks the vec4 kernel when the element count is a
+// multiple of 4 (else scalar); non-contiguous uses the stride-addressed kernel.
+static void ggml_cl_unary_ext(ggml_backend_t backend, const ggml_tensor * src0, ggml_tensor * dst,
+ cl_kernel k_f32, cl_kernel k_f32_4, cl_kernel k_f32_nc,
+ cl_kernel k_f16, cl_kernel k_f16_4, cl_kernel k_f16_nc) {
+ GGML_ASSERT(src0);
+ GGML_ASSERT(src0->extra);
+ GGML_ASSERT(dst);
+ GGML_ASSERT(dst->extra);
+
+ ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;
+
+ ggml_tensor_extra_cl * extra0 = (ggml_tensor_extra_cl *)src0->extra;
+ ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra;
+
+ cl_ulong offset0 = extra0->offset + src0->view_offs;
+ cl_ulong offsetd = extrad->offset + dst->view_offs;
+
+ const int ne00 = src0->ne[0], ne01 = src0->ne[1], ne02 = src0->ne[2], ne03 = src0->ne[3];
+ const cl_ulong nb00 = src0->nb[0], nb01 = src0->nb[1], nb02 = src0->nb[2], nb03 = src0->nb[3];
+ const cl_ulong nb0 = dst->nb[0], nb1 = dst->nb[1], nb2 = dst->nb[2], nb3 = dst->nb[3];
+
+ const bool is_f16 = (src0->type == GGML_TYPE_F16);
+ cl_kernel kernel;
+
+ if (ggml_is_contiguous(src0)) {
+ int n = ggml_nelements(dst);
+ if (n % 4 == 0) {
+ kernel = is_f16 ? k_f16_4 : k_f32_4;
+ n /= 4;
+ } else {
+ kernel = is_f16 ? k_f16 : k_f32;
+ }
+
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));
+
+ size_t global_work_size[] = {(size_t)n, 1, 1};
+ size_t local_work_size[] = {64, 1, 1};
+ size_t * local_work_size_ptr = local_work_size;
+ if (n % 64 != 0 && !backend_ctx->non_uniform_workgroups) {
+ local_work_size_ptr = nullptr;
+ }
+ backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size_ptr, dst);
+ } else {
+ kernel = is_f16 ? k_f16_nc : k_f32_nc;
+
+ CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));
+ CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extrad->data_device));
+ CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offsetd));
+ CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &ne00));
+ CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb00));
+ CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &nb01));
+ CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_ulong), &nb02));
+ CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb03));
+ CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb0));
+ CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb1));
+ CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb2));
+ CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb3));
+
+ int nth = 64;
+ size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03};
+ size_t local_work_size[] = {(size_t)nth, 1, 1};
+ backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
+ }
+}
+
+#define GGML_CL_UNARY_EXT_WRAP(FN, OP) \
+static void FN(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { \
+ UNUSED(src1); \
+ ggml_backend_opencl_context *c = (ggml_backend_opencl_context *)backend->context; \
+ ggml_cl_unary_ext(backend, src0, dst, c->kernel_##OP##_f32, c->kernel_##OP##_f32_4, c->kernel_##OP##_f32_nc, \
+ c->kernel_##OP##_f16, c->kernel_##OP##_f16_4, c->kernel_##OP##_f16_nc); \
+}
+
+GGML_CL_UNARY_EXT_WRAP(ggml_cl_sgn, sgn)
+GGML_CL_UNARY_EXT_WRAP(ggml_cl_step, step)
+GGML_CL_UNARY_EXT_WRAP(ggml_cl_elu, elu)
+GGML_CL_UNARY_EXT_WRAP(ggml_cl_hardswish, hardswish)
+GGML_CL_UNARY_EXT_WRAP(ggml_cl_hardsigmoid, hardsigmoid)
+GGML_CL_UNARY_EXT_WRAP(ggml_cl_floor, floor)
+GGML_CL_UNARY_EXT_WRAP(ggml_cl_ceil, ceil)
+GGML_CL_UNARY_EXT_WRAP(ggml_cl_round, round)
+GGML_CL_UNARY_EXT_WRAP(ggml_cl_trunc, trunc)
+
+#undef GGML_CL_UNARY_EXT_WRAP
+
static void ggml_cl_softplus(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
GGML_ASSERT(src0);
GGML_ASSERT(src0->extra);
@@ -15517,9 +15675,8 @@ static void ggml_cl_concat(ggml_backend_t backend, const ggml_tensor * src0, con
GGML_ASSERT(src1->extra);
GGML_ASSERT(dst);
GGML_ASSERT(dst->extra);
- GGML_ASSERT(src0->type == GGML_TYPE_F32);
- GGML_ASSERT(src1->type == GGML_TYPE_F32);
- GGML_ASSERT(dst->type == GGML_TYPE_F32);
+ GGML_ASSERT(src0->type == src1->type);
+ GGML_ASSERT(src0->type == dst->type);
ggml_backend_opencl_context *backend_ctx = (ggml_backend_opencl_context *)backend->context;
@@ -15561,9 +15718,21 @@ static void ggml_cl_concat(ggml_backend_t backend, const ggml_tensor * src0, con
int nth = MIN(64, ne0);
- const bool concat_pack = (dim == 0 && ne0 < 32);
- cl_kernel kernel = concat_pack ? backend_ctx->kernel_concat_f32_pack
- : backend_ctx->kernel_concat_f32;
+ const size_t ts = ggml_type_size(dst->type);
+ // the pack kernel copies 4-byte elements, so it is only valid for those.
+ const bool concat_pack = (dim == 0 && ne0 < 32 && ts == 4);
+ cl_kernel kernel;
+ if (concat_pack) {
+ kernel = backend_ctx->kernel_concat_b4_pack;
+ } else {
+ switch (ts) {
+ case 1: kernel = backend_ctx->kernel_concat_b1; break;
+ case 2: kernel = backend_ctx->kernel_concat_b2; break;
+ case 4: kernel = backend_ctx->kernel_concat_b4; break;
+ case 8: kernel = backend_ctx->kernel_concat_b8; break;
+ default: GGML_ABORT("unsupported concat element size: %zu", ts);
+ }
+ }
CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device));
CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0));
@@ -26021,6 +26190,38 @@ static void ggml_cl_cpy(ggml_backend_t backend, const ggml_tensor * src0, const
cl_ulong offset0 = extra0->offset + src0->view_offs;
cl_ulong offset1 = extra1->offset + src1->view_offs;
+ // A contiguous f32 -> f32 copy is a linear move. The kernel below maps one workgroup to
+ // each row, so a tensor with few long rows runs on a single compute unit; dispatch those
+ // over the whole device instead. GGML_OPENCL_CPY_FLAT=0 restores the row-mapped path.
+ static const bool cpy_flat_on = []{
+ const char * e = getenv("GGML_OPENCL_CPY_FLAT");
+ return !(e && e[0] == '0');
+ }();
+ if (cpy_flat_on && backend_ctx->kernel_cpy_f32_f32_flat != nullptr &&
+ src0t == GGML_TYPE_F32 && src1t == GGML_TYPE_F32 &&
+ ggml_is_contiguous(src0) && ggml_is_contiguous(src1) &&
+ ggml_nelements(src0) == ggml_nelements(src1)) {
+ cl_kernel k = backend_ctx->kernel_cpy_f32_f32_flat;
+ const cl_ulong nelem = (cl_ulong) ggml_nelements(src0);
+ const cl_ulong n4 = nelem / 4;
+
+ CL_CHECK(clSetKernelArg(k, 0, sizeof(cl_mem), &extra0->data_device));
+ CL_CHECK(clSetKernelArg(k, 1, sizeof(cl_ulong), &offset0));
+ CL_CHECK(clSetKernelArg(k, 2, sizeof(cl_mem), &extra1->data_device));
+ CL_CHECK(clSetKernelArg(k, 3, sizeof(cl_ulong), &offset1));
+ CL_CHECK(clSetKernelArg(k, 4, sizeof(cl_ulong), &nelem));
+ CL_CHECK(clSetKernelArg(k, 5, sizeof(cl_ulong), &n4));
+
+ // one work item per float4, plus one for the trailing scalars
+ const size_t items = (size_t) n4 + ((nelem % 4) ? 1 : 0);
+ const size_t lsz = MIN((size_t) 64, backend_ctx->max_workgroup_size);
+ size_t global_work_size[] = { ((items + lsz - 1) / lsz) * lsz, 1, 1 };
+ size_t local_work_size[] = { lsz, 1, 1 };
+
+ backend_ctx->enqueue_ndrange_kernel(k, 1, global_work_size, local_work_size, src1);
+ return;
+ }
+
cl_kernel kernel;
switch (src0t) {
@@ -27454,6 +27655,42 @@ bool ggml_cl_compute_forward(ggml_backend_t backend, struct ggml_tensor * tensor
}
func = ggml_cl_abs;
break;
+ case GGML_UNARY_OP_SGN:
+ if (!any_on_device) { return false; }
+ func = ggml_cl_sgn;
+ break;
+ case GGML_UNARY_OP_STEP:
+ if (!any_on_device) { return false; }
+ func = ggml_cl_step;
+ break;
+ case GGML_UNARY_OP_ELU:
+ if (!any_on_device) { return false; }
+ func = ggml_cl_elu;
+ break;
+ case GGML_UNARY_OP_HARDSWISH:
+ if (!any_on_device) { return false; }
+ func = ggml_cl_hardswish;
+ break;
+ case GGML_UNARY_OP_HARDSIGMOID:
+ if (!any_on_device) { return false; }
+ func = ggml_cl_hardsigmoid;
+ break;
+ case GGML_UNARY_OP_FLOOR:
+ if (!any_on_device) { return false; }
+ func = ggml_cl_floor;
+ break;
+ case GGML_UNARY_OP_CEIL:
+ if (!any_on_device) { return false; }
+ func = ggml_cl_ceil;
+ break;
+ case GGML_UNARY_OP_ROUND:
+ if (!any_on_device) { return false; }
+ func = ggml_cl_round;
+ break;
+ case GGML_UNARY_OP_TRUNC:
+ if (!any_on_device) { return false; }
+ func = ggml_cl_trunc;
+ break;
case GGML_UNARY_OP_SOFTPLUS:
if (!any_on_device) {
return false;
diff --git a/ggml/src/ggml-opencl/kernels/concat.cl b/ggml/src/ggml-opencl/kernels/concat.cl
index 2fbd7851d3..8ecf7466d6 100644
--- a/ggml/src/ggml-opencl/kernels/concat.cl
+++ b/ggml/src/ggml-opencl/kernels/concat.cl
@@ -1,56 +1,66 @@
-kernel void kernel_concat_f32(
- global const char * src0,
- ulong offset0,
- global const char * src1,
- ulong offset1,
- global char * dst,
- ulong offsetd,
- int ne00,
- int ne01,
- int ne02,
- int ne03,
- ulong nb00,
- ulong nb01,
- ulong nb02,
- ulong nb03,
- ulong nb10,
- ulong nb11,
- ulong nb12,
- ulong nb13,
- int ne0,
- ulong nb0,
- ulong nb1,
- ulong nb2,
- ulong nb3,
- int dim
-) {
- src0 = src0 + offset0;
- src1 = src1 + offset1;
- dst = dst + offsetd;
-
- const int i3 = get_group_id(2);
- const int i2 = get_group_id(1);
- const int i1 = get_group_id(0);
-
- int o[4] = {0, 0, 0, 0};
- o[dim] = dim == 0 ? ne00 : (dim == 1 ? ne01 : (dim == 2 ? ne02 : ne03));
-
- global const float * x;
-
- for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) {
- if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {
- x = (global const float *)(src0 + (i3 )*nb03 + (i2 )*nb02 + (i1 )*nb01 + (i0 )*nb00);
- } else {
- x = (global const float *)(src1 + (i3 - o[3])*nb13 + (i2 - o[2])*nb12 + (i1 - o[1])*nb11 + (i0 - o[0])*nb10);
- }
-
- global float * y = (global float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
+// concat is a pure copy, so the kernels are keyed by element byte size
+// (1/2/4/8) rather than logical type, matching the CUDA backend.
- *y = *x;
- }
+#define KERNEL_CONCAT(SUFFIX, T) \
+kernel void kernel_concat_##SUFFIX( \
+ global const char * src0, \
+ ulong offset0, \
+ global const char * src1, \
+ ulong offset1, \
+ global char * dst, \
+ ulong offsetd, \
+ int ne00, \
+ int ne01, \
+ int ne02, \
+ int ne03, \
+ ulong nb00, \
+ ulong nb01, \
+ ulong nb02, \
+ ulong nb03, \
+ ulong nb10, \
+ ulong nb11, \
+ ulong nb12, \
+ ulong nb13, \
+ int ne0, \
+ ulong nb0, \
+ ulong nb1, \
+ ulong nb2, \
+ ulong nb3, \
+ int dim \
+) { \
+ src0 = src0 + offset0; \
+ src1 = src1 + offset1; \
+ dst = dst + offsetd; \
+ \
+ const int i3 = get_group_id(2); \
+ const int i2 = get_group_id(1); \
+ const int i1 = get_group_id(0); \
+ \
+ int o[4] = {0, 0, 0, 0}; \
+ o[dim] = dim == 0 ? ne00 : (dim == 1 ? ne01 : (dim == 2 ? ne02 : ne03)); \
+ \
+ global const T * x; \
+ \
+ for (int i0 = get_local_id(0); i0 < ne0; i0 += get_local_size(0)) { \
+ if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) { \
+ x = (global const T *)(src0 + (i3 )*nb03 + (i2 )*nb02 + (i1 )*nb01 + (i0 )*nb00); \
+ } else { \
+ x = (global const T *)(src1 + (i3 - o[3])*nb13 + (i2 - o[2])*nb12 + (i1 - o[1])*nb11 + (i0 - o[0])*nb10); \
+ } \
+ \
+ global T * y = (global T *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0); \
+ \
+ *y = *x; \
+ } \
}
-kernel void kernel_concat_f32_pack(
+KERNEL_CONCAT(b1, char)
+KERNEL_CONCAT(b2, short)
+KERNEL_CONCAT(b4, int)
+KERNEL_CONCAT(b8, long)
+
+// packed variant for the common dim==0, small-ne0 case (4-byte elements only).
+kernel void kernel_concat_b4_pack(
global const char * src0,
ulong offset0,
global const char * src1,
@@ -104,14 +114,14 @@ kernel void kernel_concat_f32_pack(
o[dim] = dim == 0 ? ne00 : (dim == 1 ? ne01 : (dim == 2 ? ne02 : ne03));
for (int i0 = lane; i0 < ne0; i0 += tpr) {
- global const float * x;
+ global const int * x;
if (i0 < ne00 && i1 < ne01 && i2 < ne02 && i3 < ne03) {
- x = (global const float *)(src0 + (i3 )*nb03 + (i2 )*nb02 + (i1 )*nb01 + (i0 )*nb00);
+ x = (global const int *)(src0 + (i3 )*nb03 + (i2 )*nb02 + (i1 )*nb01 + (i0 )*nb00);
} else {
- x = (global const float *)(src1 + (i3 - o[3])*nb13 + (i2 - o[2])*nb12 + (i1 - o[1])*nb11 + (i0 - o[0])*nb10);
+ x = (global const int *)(src1 + (i3 - o[3])*nb13 + (i2 - o[2])*nb12 + (i1 - o[1])*nb11 + (i0 - o[0])*nb10);
}
- global float * y = (global float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
+ global int * y = (global int *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
*y = *x;
}
diff --git a/ggml/src/ggml-opencl/kernels/cpy.cl b/ggml/src/ggml-opencl/kernels/cpy.cl
index adbd2e766d..e875bfaf75 100644
--- a/ggml/src/ggml-opencl/kernels/cpy.cl
+++ b/ggml/src/ggml-opencl/kernels/cpy.cl
@@ -286,3 +286,28 @@ kernel void kernel_cpy_i32_i32(
dst_data[i00] = src[0];
}
}
+
+// Contiguous f32 copy, one work item per float4 over the whole tensor. The kernels above map
+// one workgroup to each row, which leaves a tensor with few long rows on a single compute unit.
+// vload4/vstore4 rather than a float4 cast: these buffers carry an arbitrary 4-byte view offset.
+kernel void kernel_cpy_f32_f32_flat(
+ global float * src0,
+ ulong offset0,
+ global float * dst,
+ ulong offsetd,
+ ulong ne,
+ ulong n4
+) {
+ src0 = (global float*)((global char*)src0 + offset0);
+ dst = (global float*)((global char*)dst + offsetd);
+
+ const ulong i = get_global_id(0);
+
+ if (i < n4) {
+ vstore4(vload4(i, src0), i, dst);
+ } else if (i == n4) {
+ for (ulong t = n4 * 4; t < ne; ++t) {
+ dst[t] = src0[t];
+ }
+ }
+}
diff --git a/ggml/src/ggml-opencl/kernels/unary_ext.cl b/ggml/src/ggml-opencl/kernels/unary_ext.cl
new file mode 100644
index 0000000000..e86eadfa5f
--- /dev/null
+++ b/ggml/src/ggml-opencl/kernels/unary_ext.cl
@@ -0,0 +1,85 @@
+#pragma OPENCL EXTENSION cl_khr_fp16 : enable
+
+//------------------------------------------------------------------------------
+// Extended elementwise unary ops, same variant shape as abs.cl:
+// f32, f32_4 (vec4), f16, f16_4 (vec4), f32_nc, f16_nc (stride-addressed).
+//
+// sgn, step, elu, hardswish, hardsigmoid, floor, ceil, round, trunc.
+//
+// Semantics match the ggml CPU reference (ggml.c). Values are computed in float
+// (the f16 variants read/write half and convert), so the conditional ops match
+// the CPU bit-for-bit within tolerance. SEXPR is the scalar form, VEXPR the
+// float4 form (vector ternaries need select()).
+//------------------------------------------------------------------------------
+
+#define UNARY_EXT(NAME, SEXPR, VEXPR) \
+kernel void kernel_##NAME##_f32( \
+ global const float * src0, ulong offset0, \
+ global float * dst, ulong offsetd) { \
+ src0 = (global float*)((global char*)src0 + offset0); \
+ dst = (global float*)((global char*)dst + offsetd); \
+ float x = src0[get_global_id(0)]; \
+ dst[get_global_id(0)] = (SEXPR); \
+} \
+kernel void kernel_##NAME##_f32_4( \
+ global const float4 * src0, ulong offset0, \
+ global float4 * dst, ulong offsetd) { \
+ src0 = (global float4*)((global char*)src0 + offset0); \
+ dst = (global float4*)((global char*)dst + offsetd); \
+ float4 x = src0[get_global_id(0)]; \
+ dst[get_global_id(0)] = (VEXPR); \
+} \
+kernel void kernel_##NAME##_f16( \
+ global const half * src0, ulong offset0, \
+ global half * dst, ulong offsetd) { \
+ src0 = (global half*)((global char*)src0 + offset0); \
+ dst = (global half*)((global char*)dst + offsetd); \
+ float x = src0[get_global_id(0)]; \
+ dst[get_global_id(0)] = (SEXPR); \
+} \
+kernel void kernel_##NAME##_f16_4( \
+ global const half4 * src0, ulong offset0, \
+ global half4 * dst, ulong offsetd) { \
+ src0 = (global half4*)((global char*)src0 + offset0); \
+ dst = (global half4*)((global char*)dst + offsetd); \
+ float4 x = convert_float4(src0[get_global_id(0)]); \
+ dst[get_global_id(0)] = convert_half4(VEXPR); \
+} \
+kernel void kernel_##NAME##_f32_nc( \
+ global const char * src0, ulong offset0, \
+ global char * dst, ulong offsetd, \
+ int ne00, ulong nb00, ulong nb01, ulong nb02, ulong nb03, \
+ ulong nb0, ulong nb1, ulong nb2, ulong nb3) { \
+ src0 = src0 + offset0; dst = dst + offsetd; \
+ const int i3 = get_group_id(2); \
+ const int i2 = get_group_id(1); \
+ const int i1 = get_group_id(0); \
+ for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) { \
+ float x = *(global const float *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00); \
+ *(global float *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0) = (SEXPR); \
+ } \
+} \
+kernel void kernel_##NAME##_f16_nc( \
+ global const char * src0, ulong offset0, \
+ global char * dst, ulong offsetd, \
+ int ne00, ulong nb00, ulong nb01, ulong nb02, ulong nb03, \
+ ulong nb0, ulong nb1, ulong nb2, ulong nb3) { \
+ src0 = src0 + offset0; dst = dst + offsetd; \
+ const int i3 = get_group_id(2); \
+ const int i2 = get_group_id(1); \
+ const int i1 = get_group_id(0); \
+ for (int i0 = get_local_id(0); i0 < ne00; i0 += get_local_size(0)) {\
+ float x = *(global const half *)(src0 + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00); \
+ *(global half *)(dst + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0) = (SEXPR); \
+ } \
+}
+
+UNARY_EXT(sgn, sign(x), sign(x))
+UNARY_EXT(step, x > 0.0f ? 1.0f : 0.0f, select((float4)0.0f, (float4)1.0f, x > 0.0f))
+UNARY_EXT(elu, x > 0.0f ? x : expm1(x), select(expm1(x), x, x > 0.0f))
+UNARY_EXT(hardswish, x * fmin(1.0f, fmax(0.0f, (x + 3.0f) / 6.0f)), x * fmin((float4)1.0f, fmax((float4)0.0f, (x + 3.0f) / 6.0f)))
+UNARY_EXT(hardsigmoid, fmin(1.0f, fmax(0.0f, (x + 3.0f) / 6.0f)), fmin((float4)1.0f, fmax((float4)0.0f, (x + 3.0f) / 6.0f)))
+UNARY_EXT(floor, floor(x), floor(x))
+UNARY_EXT(ceil, ceil(x), ceil(x))
+UNARY_EXT(round, round(x), round(x))
+UNARY_EXT(trunc, trunc(x), trunc(x))