LLM inference in C/C++
C++ ggml
metal : add SILU_BACK (#25982)
Talha Adnan221f0f6356ef
diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cppindex ec3ac5140e..e5ef797555 100644--- a/ggml/src/ggml-metal/ggml-metal-device.cpp+++ b/ggml/src/ggml-metal/ggml-metal-device.cpp@@ -2135,6 +2135,23 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_opt_step_sgd(ggmreturn res;}+ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_silu_back(ggml_metal_library_t lib, const ggml_tensor * op) {+ assert(op->op == GGML_OP_SILU_BACK);++ char base[256];+ char name[256];++ snprintf(base, 256, "kernel_silu_back_%s", ggml_type_name(op->src[0]->type));+ snprintf(name, 256, "%s", base);++ ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);+ if (!res.pipeline) {+ res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);+ }++ return res;+}+ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_memset(ggml_metal_library_t lib, const ggml_tensor * op) {GGML_ASSERT(op->type == GGML_TYPE_I64);diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.hindex 2d712f0bfc..63e6d0c8aa 100644--- a/ggml/src/ggml-metal/ggml-metal-device.h+++ b/ggml/src/ggml-metal/ggml-metal-device.h@@ -117,6 +117,7 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_diagstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_repeat (ggml_metal_library_t lib, enum ggml_type tsrc);struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_concat (ggml_metal_library_t lib, enum ggml_type tsrc);struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_unary (ggml_metal_library_t lib, const struct ggml_tensor * op);+struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_silu_back (ggml_metal_library_t lib, const struct ggml_tensor * op);struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_glu (ggml_metal_library_t lib, const struct ggml_tensor * op);struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_sum (ggml_metal_library_t lib, const struct ggml_tensor * op);struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_sum_rows (ggml_metal_library_t lib, const struct ggml_tensor * op);diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.mindex 20a2fdcfa4..827d7ca320 100644--- a/ggml/src/ggml-metal/ggml-metal-device.m+++ b/ggml/src/ggml-metal/ggml-metal-device.m@@ -1137,6 +1137,14 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_tedefault:return false;}+ case GGML_OP_SILU_BACK:+ return (op->src[0]->type == GGML_TYPE_F32) &&+ (op->src[1]->type == GGML_TYPE_F32) &&+ (op->type == GGML_TYPE_F32) &&+ ggml_is_contiguous(op->src[0]) &&+ ggml_is_contiguous(op->src[1]) &&+ ggml_is_contiguous(op) &&+ ggml_are_same_shape(op->src[0], op->src[1]);case GGML_OP_GLU:switch (ggml_get_glu_op(op)) {case GGML_GLU_OP_REGLU:diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.hindex 0fa0c2d6ad..8e8926cda4 100644--- a/ggml/src/ggml-metal/ggml-metal-impl.h+++ b/ggml/src/ggml-metal/ggml-metal-impl.h@@ -1265,4 +1265,8 @@ typedef struct {int64_t np;} ggml_metal_kargs_opt_step_sgd;+typedef struct {+ int64_t ne;+} ggml_metal_kargs_silu_back;+#endif // GGML_METAL_IMPLdiff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cppindex caa162a5c9..2d84804b92 100644--- a/ggml/src/ggml-metal/ggml-metal-ops.cpp+++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp@@ -299,6 +299,10 @@ static int ggml_metal_op_encode_impl(ggml_metal_op_t ctx, int idx) {{n_fuse = ggml_metal_op_unary(ctx, idx);} break;+ case GGML_OP_SILU_BACK:+ {+ n_fuse = ggml_metal_op_silu_back(ctx, idx);+ } break;case GGML_OP_GLU:{n_fuse = ggml_metal_op_glu(ctx, idx);@@ -3473,6 +3477,36 @@ int ggml_metal_op_bin(ggml_metal_op_t ctx, int idx) {return n_fuse;}+int ggml_metal_op_silu_back(ggml_metal_op_t ctx, int idx) {+ ggml_tensor * op = ctx->node(idx);++ ggml_metal_library_t lib = ctx->lib;+ ggml_metal_encoder_t enc = ctx->enc;++ auto pipeline = ggml_metal_library_get_pipeline_silu_back(lib, op);++ const int64_t ne = ggml_nelements(op);++ ggml_metal_kargs_silu_back args = {+ /*.ne =*/ ne,+ };++ int arg_idx{0};++ ggml_metal_encoder_set_pipeline(enc, pipeline);+ ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), arg_idx++);+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), arg_idx++);+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), arg_idx++);+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), arg_idx++);++ const int nth = std::min<int64_t>(ggml_metal_pipeline_max_theads_per_threadgroup(pipeline), ne);+ const int64_t n = (ne + nth - 1) / nth;++ ggml_metal_encoder_dispatch_threadgroups(enc, n, 1, 1, nth, 1, 1);++ return 1;+}+int ggml_metal_op_l2_norm(ggml_metal_op_t ctx, int idx) {ggml_tensor * op = ctx->node(idx);diff --git a/ggml/src/ggml-metal/ggml-metal-ops.h b/ggml/src/ggml-metal/ggml-metal-ops.hindex bd20bdfbcd..4fdad1dd59 100644--- a/ggml/src/ggml-metal/ggml-metal-ops.h+++ b/ggml/src/ggml-metal/ggml-metal-ops.h@@ -71,6 +71,7 @@ int ggml_metal_op_mul_mat_id (ggml_metal_op_t ctx, int idx);int ggml_metal_op_add_id (ggml_metal_op_t ctx, int idx);int ggml_metal_op_flash_attn_ext (ggml_metal_op_t ctx, int idx);int ggml_metal_op_bin (ggml_metal_op_t ctx, int idx);+int ggml_metal_op_silu_back (ggml_metal_op_t ctx, int idx);int ggml_metal_op_l2_norm (ggml_metal_op_t ctx, int idx);int ggml_metal_op_group_norm (ggml_metal_op_t ctx, int idx);int ggml_metal_op_norm (ggml_metal_op_t ctx, int idx);diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metalindex 6044241193..47d4f685a0 100644--- a/ggml/src/ggml-metal/ggml-metal.metal+++ b/ggml/src/ggml-metal/ggml-metal.metal@@ -1255,6 +1255,20 @@ template [[host_name("kernel_unary_f32_f32_4")]] kernel kernel_unary_t kernel_untemplate [[host_name("kernel_unary_f16_f16")]] kernel kernel_unary_t kernel_unary_impl<half, half, float>;template [[host_name("kernel_unary_f16_f16_4")]] kernel kernel_unary_t kernel_unary_impl<half4, half4, float4>;+kernel void kernel_silu_back_f32(+ constant ggml_metal_kargs_silu_back & args,+ device const float * dy,+ device const float * x,+ device float * dx,+ uint gid [[thread_position_in_grid]]) {+ if (gid >= args.ne) {+ return;+ }++ const float s = 1.0f / (1.0f + exp(-x[gid]));+ dx[gid] = dy[gid] * s * (1.0f + x[gid] * (1.0f - s));+}+// OP: 0 - add, 1 - sub, 2 - mul, 3 - divconstant short FC_bin_op [[function_constant(FC_BIN + 0)]];constant short FC_bin_f [[function_constant(FC_BIN + 1)]];