LLM inference in C/C++
C++ ggml
opencl: add get_rows for i32, cpy for i32 and f32 (#30756)
lhez6fa4f6cd3ef4
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cppindex de93b0134e..96461bcd94 100644--- a/ggml/src/ggml-opencl/ggml-opencl.cpp+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp@@ -868,7 +868,7 @@ struct ggml_backend_opencl_context {#ifdef GGML_OPENCL_USE_ADRENO_KERNELSggml_cl_adreno_xmem_attn_state adreno_xmem_attn;#endif- cl_kernel kernel_get_rows_f32, kernel_get_rows_f16, kernel_get_rows_q4_0;+ cl_kernel kernel_get_rows_f32, kernel_get_rows_f16, kernel_get_rows_i32, kernel_get_rows_q4_0;cl_kernel kernel_set_rows_f32_i64, kernel_set_rows_f32_i32, kernel_set_rows_f16_i64, kernel_set_rows_f16_i32;cl_kernel kernel_set_rows_q8_0_i64, kernel_set_rows_q8_0_i32;cl_kernel kernel_set_rows_q8_0_soa_i64, kernel_set_rows_q8_0_soa_i32;@@ -876,7 +876,7 @@ struct ggml_backend_opencl_context {cl_kernel kernel_set_rows_q4_0_soa_i64, kernel_set_rows_q4_0_soa_i32;cl_kernel kernel_rope_norm_f32, kernel_rope_norm_f16, kernel_rope_neox_f32, kernel_rope_neox_f16;cl_kernel kernel_rope_multi_f32, kernel_rope_multi_f16, kernel_rope_vision_f32, kernel_rope_vision_f16;- cl_kernel kernel_cpy_f16_f16, kernel_cpy_f16_f32, kernel_cpy_f32_f16, kernel_cpy_f32_f32, kernel_cpy_i32_i32;+ cl_kernel kernel_cpy_f16_f16, kernel_cpy_f16_f32, kernel_cpy_f32_f16, kernel_cpy_f32_f32, kernel_cpy_i32_i32, kernel_cpy_i32_f32, kernel_cpy_f32_i32;cl_kernel kernel_cpy_f32_f32_pack = nullptr;cl_kernel kernel_cpy_f32_f32_flat = nullptr;cl_kernel kernel_mul_mat_f32_f32;@@ -1652,6 +1652,8 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {}}CL_CHECK((backend_ctx->kernel_cpy_i32_i32 = clCreateKernel(prog, "kernel_cpy_i32_i32", &err), err));+ CL_CHECK((backend_ctx->kernel_cpy_i32_f32 = clCreateKernel(prog, "kernel_cpy_i32_f32", &err), err));+ CL_CHECK((backend_ctx->kernel_cpy_f32_i32 = clCreateKernel(prog, "kernel_cpy_f32_i32", &err), err));GGML_LOG_CONT(".");}@@ -1840,6 +1842,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts);CL_CHECK((backend_ctx->kernel_get_rows_f32 = clCreateKernel(backend_ctx->program_get_rows, "kernel_get_rows_f32", &err), err));+ CL_CHECK((backend_ctx->kernel_get_rows_i32 = clCreateKernel(backend_ctx->program_get_rows, "kernel_get_rows_i32", &err), err));CL_CHECK((backend_ctx->kernel_get_rows_f16 = clCreateKernel(backend_ctx->program_get_rows, "kernel_get_rows_f16", &err), err));CL_CHECK((backend_ctx->kernel_get_rows_q4_0 = clCreateKernel(backend_ctx->program_get_rows, "kernel_get_rows_q4_0", &err), err));GGML_LOG_CONT(".");@@ -9102,6 +9105,8 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_tereturn true;case GGML_OP_GET_ROWS:switch (op->src[0]->type) {+ case GGML_TYPE_I32:+ return op->type == GGML_TYPE_I32;case GGML_TYPE_F32:case GGML_TYPE_F16:return true;@@ -9139,6 +9144,7 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_teswitch (op->src[0]->type) {case GGML_TYPE_F32:switch (op->type) {+ case GGML_TYPE_I32:case GGML_TYPE_F16:case GGML_TYPE_F32:return true;@@ -9155,6 +9161,7 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te}case GGML_TYPE_I32:switch (op->type) {+ case GGML_TYPE_F32:case GGML_TYPE_I32:return true;default:@@ -13619,6 +13626,9 @@ static void ggml_cl_get_rows(ggml_backend_t backend, const ggml_tensor * src0, ccase GGML_TYPE_F32:kernel = backend_ctx->kernel_get_rows_f32;break;+ case GGML_TYPE_I32:+ kernel = backend_ctx->kernel_get_rows_i32;+ break;case GGML_TYPE_F16:kernel = backend_ctx->kernel_get_rows_f16;break;@@ -13654,7 +13664,7 @@ static void ggml_cl_get_rows(ggml_backend_t backend, const ggml_tensor * src0, c}int nchunks = 1;- if (src0->type == GGML_TYPE_F32) {+ if (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_I32) {const int chunk_target = nth * 4;nchunks = (ne00 + chunk_target - 1) / chunk_target;nchunks = MAX(1, MIN(nchunks, 64));@@ -28726,6 +28736,9 @@ static void ggml_cl_cpy(ggml_backend_t backend, const ggml_tensor * src0, constswitch (src0t) {case GGML_TYPE_F32:switch (src1t) {+ case GGML_TYPE_I32:+ kernel = backend_ctx->kernel_cpy_f32_i32;+ break;case GGML_TYPE_F16:kernel = backend_ctx->kernel_cpy_f32_f16;break;@@ -28752,6 +28765,9 @@ static void ggml_cl_cpy(ggml_backend_t backend, const ggml_tensor * src0, constbreak;case GGML_TYPE_I32:switch (src1t) {+ case GGML_TYPE_F32:+ kernel = backend_ctx->kernel_cpy_i32_f32;+ break;case GGML_TYPE_I32:kernel = backend_ctx->kernel_cpy_i32_i32;break;diff --git a/ggml/src/ggml-opencl/kernels/cpy.cl b/ggml/src/ggml-opencl/kernels/cpy.clindex 11d7a57c17..18400d37f1 100644--- a/ggml/src/ggml-opencl/kernels/cpy.cl+++ b/ggml/src/ggml-opencl/kernels/cpy.cl@@ -313,3 +313,88 @@ kernel void kernel_cpy_f32_f32_flat(}}}++kernel void kernel_cpy_i32_f32(+ global int * src0,+ ulong offset0,+ global float * dst,+ ulong offsetd,+ int ne00,+ int ne01,+ int ne02,+ int ne03,+ ulong nb00,+ ulong nb01,+ ulong nb02,+ ulong nb03,+ int ne0,+ int ne1,+ int ne2,+ int ne3,+ ulong nb0,+ ulong nb1,+ ulong nb2,+ ulong nb3+) {+ src0 = (global int *) ((global char *) src0 + offset0);+ dst = (global float *) ((global char *) dst + offsetd);++ const ulong i01 = get_group_id(0);+ const ulong i02 = get_group_id(1);+ const ulong i03 = get_group_id(2);++ for (ulong i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) {+ const ulong n = ((i03 * ne02 + i02) * ne01 + i01) * ne00 + i00;+ const ulong i0 = n % ne0;+ const ulong i1 = n / ne0 % ne1;+ const ulong i2 = n / ne0 / ne1 % ne2;+ const ulong i3 = n / ne0 / ne1 / ne2;++ global const int * src = (global const int *) ((global char *) src0 + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);+ global float * out = (global float *) ((global char *) dst + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);+ out[0] = convert_float(src[0]);+ }+}+++kernel void kernel_cpy_f32_i32(+ global float * src0,+ ulong offset0,+ global int * dst,+ ulong offsetd,+ int ne00,+ int ne01,+ int ne02,+ int ne03,+ ulong nb00,+ ulong nb01,+ ulong nb02,+ ulong nb03,+ int ne0,+ int ne1,+ int ne2,+ int ne3,+ ulong nb0,+ ulong nb1,+ ulong nb2,+ ulong nb3+) {+ src0 = (global float *) ((global char *) src0 + offset0);+ dst = (global int *) ((global char *) dst + offsetd);++ const ulong i01 = get_group_id(0);+ const ulong i02 = get_group_id(1);+ const ulong i03 = get_group_id(2);++ for (ulong i00 = get_local_id(0); i00 < ne00; i00 += get_local_size(0)) {+ const ulong n = ((i03 * ne02 + i02) * ne01 + i01) * ne00 + i00;+ const ulong i0 = n % ne0;+ const ulong i1 = n / ne0 % ne1;+ const ulong i2 = n / ne0 / ne1 % ne2;+ const ulong i3 = n / ne0 / ne1 / ne2;++ global const float * src = (global const float *) ((global char *) src0 + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);+ global int * out = (global int *) ((global char *) dst + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3);+ out[0] = convert_int_rtz(src[0]);+ }+}diff --git a/ggml/src/ggml-opencl/kernels/get_rows.cl b/ggml/src/ggml-opencl/kernels/get_rows.clindex 9ae4fff09f..f060e46aec 100644--- a/ggml/src/ggml-opencl/kernels/get_rows.cl+++ b/ggml/src/ggml-opencl/kernels/get_rows.cl@@ -106,6 +106,53 @@ kernel void kernel_get_rows_f32(}}+kernel void kernel_get_rows_i32(+ global void * src0,+ ulong offset0,+ global int * src1,+ ulong offset1,+ global int * dst,+ ulong offsetd,+ int ne00,+ ulong nb01,+ ulong nb02,+ ulong nb03,+ int ne10,+ ulong nb10,+ ulong nb11,+ ulong nb12,+ ulong nb1,+ ulong nb2,+ ulong nb3+) {+ src0 = (global void*)((global char*)src0 + offset0);+ src1 = (global int*)((global char*)src1 + offset1);+ dst = (global int*)((global char*)dst + offsetd);++ int nchunks = get_num_groups(0) / ne10;+ int g = get_group_id(0);+ int i10 = g / nchunks;+ int chunk = g - i10 * nchunks;+ int i11 = get_group_id(1);+ int i12 = get_group_id(2);++ int r = ((global int *) ((global char *) src1 + i12*nb12 + i11*nb11 + i10*nb10))[0];++ int i02 = i11;+ int i03 = i12;++ global int * dst_row = (global int *) ((global char *) dst + i12*nb3 + i11*nb2 + i10*nb1);+ global int * src_row = (global int *) ((global char *) src0 + r*nb01 + i02*nb02 + i03*nb03);++ int span = (ne00 + nchunks - 1) / nchunks;+ int start = chunk * span;+ int end = min(start + span, ne00);++ for (int ind = start + get_local_id(0); ind < end; ind += get_local_size(0)) {+ dst_row[ind] = src_row[ind];+ }+}+kernel void kernel_get_rows_f16(global void * src0,ulong offset0,