LLM inference in C/C++
C++ ggml
vulkan : added the PAD_REFLECT_1D operation (#26586)
Safi Ullah3f545beccee6
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cppindex f6cbaecb7f..c1d86aaac5 100644--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp@@ -955,6 +955,7 @@ struct vk_device_struct {vk_pipeline pipeline_diag[2];vk_pipeline pipeline_clamp[2];vk_pipeline pipeline_pad_f32;+ vk_pipeline pipeline_pad_reflect_1d_f32;vk_pipeline pipeline_roll_f32;vk_pipeline pipeline_repeat_i32, pipeline_repeat_back_f32;vk_pipeline pipeline_repeat_i16;@@ -5630,6 +5631,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {ggml_vk_create_pipeline(device, device->pipeline_diag[1], "diag_f16", diag_f16_len, diag_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_pad_f32, "pad_f32", pad_f32_len, pad_f32_data, "main", 2, sizeof(vk_op_pad_push_constants), {512, 1, 1}, {}, 1);+ ggml_vk_create_pipeline(device, device->pipeline_pad_reflect_1d_f32, "pad_reflect_1d_f32", pad_reflect_1d_f32_len, pad_reflect_1d_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_roll_f32, "roll_f32", roll_f32_len, roll_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);@@ -11336,6 +11338,11 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, constreturn ctx->device->pipeline_pad_f32;}return nullptr;+ case GGML_OP_PAD_REFLECT_1D:+ if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {+ return ctx->device->pipeline_pad_reflect_1d_f32;+ }+ return nullptr;case GGML_OP_ROLL:if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {return ctx->device->pipeline_roll_f32;@@ -12239,6 +12246,7 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, cocase GGML_OP_CLAMP:case GGML_OP_LEAKY_RELU:case GGML_OP_PAD:+ case GGML_OP_PAD_REFLECT_1D:case GGML_OP_ROLL:case GGML_OP_REPEAT:case GGML_OP_REPEAT_BACK:@@ -13111,6 +13119,17 @@ static void ggml_vk_pad(ggml_backend_vk_context * ctx, vk_context& subctx, constggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_PAD, std::move(p));}+static void ggml_vk_pad_reflect_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {+ const uint32_t p0 = (uint32_t)dst->op_params[0];+ const uint32_t p1 = (uint32_t)dst->op_params[1];++ vk_op_unary_push_constants p = vk_op_unary_push_constants_init(src0, dst, ggml_nelements(dst));+ memcpy(&p.param1, &p0, sizeof(float));+ memcpy(&p.param2, &p1, sizeof(float));++ ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_PAD_REFLECT_1D, std::move(p));+}+static void ggml_vk_roll(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {const int32_t s0 = ggml_get_op_params_i32(dst, 0);const int32_t s1 = ggml_get_op_params_i32(dst, 1);@@ -15520,6 +15539,10 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgrcase GGML_OP_PAD:ggml_vk_pad(ctx, compute_ctx, src0, node);+ break;+ case GGML_OP_PAD_REFLECT_1D:+ ggml_vk_pad_reflect_1d(ctx, compute_ctx, src0, node);+break;case GGML_OP_ROLL:ggml_vk_roll(ctx, compute_ctx, src0, node);@@ -18446,6 +18469,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggmcase GGML_OP_SCALE:return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32;case GGML_OP_PAD:+ case GGML_OP_PAD_REFLECT_1D:case GGML_OP_ROLL:return op->src[0]->type == GGML_TYPE_F32;case GGML_OP_DIAG_MASK_INF:@@ -19228,6 +19252,8 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_cgraph *} else if (tensor->op == GGML_OP_PAD) {tensor_clone = ggml_pad_ext(ggml_ctx, src_clone[0], tensor->op_params[0], tensor->op_params[1], tensor->op_params[2], tensor->op_params[3],tensor->op_params[4], tensor->op_params[5], tensor->op_params[6], tensor->op_params[7]);+ } else if (tensor->op == GGML_OP_PAD_REFLECT_1D) {+ tensor_clone = ggml_pad_reflect_1d(ggml_ctx, src_clone[0], tensor->op_params[0], tensor->op_params[1]);} else if (tensor->op == GGML_OP_REPEAT) {tensor_clone = ggml_repeat(ggml_ctx, src_clone[0], tensor);} else if (tensor->op == GGML_OP_REPEAT_BACK) {diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/pad_reflect_1d.comp b/ggml/src/ggml-vulkan/vulkan-shaders/pad_reflect_1d.compnew file mode 100644index 0000000000..2389020fae--- /dev/null+++ b/ggml/src/ggml-vulkan/vulkan-shaders/pad_reflect_1d.comp@@ -0,0 +1,43 @@+#version 450++#include "types.glsl"+#include "generic_unary_head.glsl" // included to use functions like fastdiv etc.++layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;++void main() {++ const uint idx = get_idx();++ if (idx >= p.ne) {+ return;+ }++ const uint p0 = floatBitsToUint(p.param1);+ const uint p1 = floatBitsToUint(p.param2);++ const uint i3 = fastdiv(idx, p.ne1_012mp, fastdiv_L(p.ne1_Ls, 0));+ const uint i3_offset = i3 * p.ne12 * p.ne11 * p.ne10;++ const uint i2 = fastdiv(idx - i3_offset, p.ne1_01mp, fastdiv_L(p.ne1_Ls, 1));+ const uint i2_offset = i2 * p.ne11 * p.ne10;++ const uint i1 = fastdiv(idx - i3_offset - i2_offset, p.ne1_0mp, fastdiv_L(p.ne1_Ls, 2));+ const uint i0 = idx - i3_offset - i2_offset - i1 * p.ne10;++ uint src_col;++ if (i0 < p0) {+ src_col = p0 - i0; // left pad area+ } else if (i0 < p0 + p.ne00) {+ src_col = i0 - p0; // center area+ } else {+ src_col = 2u * p.ne00 - 2u - (i0 - p0); // right pad area+ }++ const uint src_idx = i3 * p.nb03 + i2 * p.nb02 + i1 * p.nb01 + src_col * p.nb00;+ const uint d_idx = i3 * p.nb13 + i2 * p.nb12 + i1 * p.nb11 + i0 * p.nb10;++ // copy the computed value to the destination tensor+ data_d[get_doffset() + d_idx] = D_TYPE(data_a[get_aoffset() + src_idx]);+}diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cppindex caa0c889a4..17d57d5a18 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp+++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp@@ -896,6 +896,7 @@ void process_shaders() {string_to_spv("scale_f32", "scale.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});string_to_spv("pad_f32", "pad.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});+ string_to_spv("pad_reflect_1d_f32", "pad_reflect_1d.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});string_to_spv("concat_i8", "concat.comp", {{"A_TYPE", "uint8_t"}, {"B_TYPE", "uint8_t"}, {"D_TYPE", "uint8_t"}});string_to_spv("concat_i16", "concat.comp", {{"A_TYPE", "uint16_t"}, {"B_TYPE", "uint16_t"}, {"D_TYPE", "uint16_t"}});