LLM inference in C/C++
C++ ggml
vulkan : fuse UNARY(GELU|SIGMOID|SILU|SOFTPLUS) + MUL (#27220)
Ankit Khandelwal64e9bceb2c3a
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cppindex 738e7cd922..8f37f65b8a 100644--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp@@ -1071,6 +1071,9 @@ struct vk_device_struct {vk_pipeline pipeline_trunc[2];vk_pipeline pipeline_sgn[2];+ // fused UNARY+MUL pipelines: [op][f16][norepeat][op_on_b]+ vk_pipeline pipeline_unary_mul[4][2][2][2];+vk_pipeline pipeline_add1_f16_f16;vk_pipeline pipeline_add1_f16_f32;vk_pipeline pipeline_add1_f32_f32;@@ -5930,6 +5933,26 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {CREATE_UNARY(expm1)#undef CREATE_UNARY+// spec constants: {norepeat, op_on_b}+#define CREATE_UNARY_MUL(name, idx) \+ for (int dt = 0; dt < 2; ++dt) { \+ const size_t len_ = dt ? name ## _mul_f16_len : name ## _mul_f32_len; \+ const unsigned char * data_ = dt ? name ## _mul_f16_data : name ## _mul_f32_data; \+ const std::string dts_ = dt ? "f16" : "f32"; \+ for (int ob = 0; ob < 2; ++ob) \+ for (int nr = 0; nr < 2; ++nr) \+ ggml_vk_create_pipeline(device, device->pipeline_unary_mul[(idx)][dt][nr][ob], \+ (#name "_mul" + std::string(ob ? "_b" : "") + "_" + dts_ + (nr ? "_norepeat" : "")).c_str(), \+ len_, data_, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, \+ { (uint32_t) nr, (uint32_t) ob }, 1); \+ }++ CREATE_UNARY_MUL(gelu, 0)+ CREATE_UNARY_MUL(sigmoid, 1)+ CREATE_UNARY_MUL(silu, 2)+ CREATE_UNARY_MUL(softplus, 3)+#undef CREATE_UNARY_MUL+ggml_vk_create_pipeline(device, device->pipeline_add1_f16_f16, "add1_f16_f16", add1_f16_f16_len, add1_f16_f16_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_add1_f16_f32, "add1_f16_f32", add1_f16_f32_len, add1_f16_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1);ggml_vk_create_pipeline(device, device->pipeline_add1_f32_f32, "add1_f32_f32", add1_f32_f32_len, add1_f32_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1);@@ -12416,7 +12439,7 @@ template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk}template<typename PC>-static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst, ggml_op op, PC&& pc) {+static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst, ggml_op op, PC&& pc, vk_pipeline pipeline_override = nullptr) {VK_LOG_DEBUG("ggml_vk_op_f32((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3];if (src1 != nullptr) {std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3];@@ -12447,7 +12470,12 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, coinit_pushconst_fastdiv(pc);- vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, src0, src1, src2, dst, op);+ vk_pipeline pipeline;+ if (pipeline_override) {+ pipeline = pipeline_override;+ } else {+ pipeline = ggml_vk_op_get_pipeline(ctx, src0, src1, src2, dst, op);+ }if (pipeline == nullptr) {std::cerr << "ggml_vulkan: Error: Missing op: " << ggml_op_name(op) << " for " << ggml_type_name(src0->type);@@ -13032,6 +13060,52 @@ static void ggml_vk_mul(ggml_backend_vk_context * ctx, vk_context& subctx, const});}+// index into device->pipeline_unary_mul for the supported unary ops, or -1+static int ggml_vk_unary_mul_op_index(ggml_unary_op op) {+ switch (op) {+ case GGML_UNARY_OP_GELU: return 0;+ case GGML_UNARY_OP_SIGMOID: return 1;+ case GGML_UNARY_OP_SILU: return 2;+ case GGML_UNARY_OP_SOFTPLUS: return 3;+ default: return -1;+ }+}++static void ggml_vk_unary_mul(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) {+ const ggml_tensor * unary = cgraph->nodes[node_idx];+ ggml_tensor * mul = cgraph->nodes[node_idx + 1];++ // unary on src1 that tiles into src0+ const bool op_on_b = mul->src[1] == unary &&+ !ggml_are_same_shape(unary->src[0], mul->src[0]) &&+ ggml_can_repeat(unary, mul->src[0]);++ const ggml_tensor * src0 = op_on_b ? mul->src[0] : unary->src[0];+ const ggml_tensor * src1 = op_on_b ? unary->src[0] :+ ((mul->src[0] == unary) ? mul->src[1] : mul->src[0]);++ const bool f16 = src0->type == GGML_TYPE_F16;+ const bool norepeat = ggml_are_same_shape(src0, src1);+ const int oi = ggml_vk_unary_mul_op_index(ggml_get_unary_op(unary));+ if (oi < 0) {+ GGML_ABORT("fatal error");+ }+ vk_pipeline pipeline = ctx->device->pipeline_unary_mul[oi][f16][norepeat][op_on_b];++ const uint32_t src0_type_size = ggml_type_size(src0->type);+ const uint32_t src1_type_size = ggml_type_size(src1->type);+ const uint32_t dst_type_size = ggml_type_size(mul->type);++ ggml_vk_op_f32<vk_op_binary_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, mul, GGML_OP_UNARY, {+ (uint32_t)ggml_nelements(op_on_b ? mul : src0),+ (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size,+ (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size,+ (uint32_t) mul->ne[0], (uint32_t) mul->ne[1], (uint32_t) mul->ne[2],(uint32_t) mul->ne[3], (uint32_t) mul->nb[0] / dst_type_size, (uint32_t) mul->nb[1] / dst_type_size, (uint32_t) mul->nb[2] / dst_type_size, (uint32_t) mul->nb[3] / dst_type_size,+ 0,+ 0.0f, 0.0f, 0,+ }, pipeline);+}+static void ggml_vk_div(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {const uint32_t src0_type_size = ggml_type_size(src0->type);const uint32_t src1_type_size = ggml_type_size(src1->type);@@ -16314,6 +16388,10 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgrggml_vk_topk_moe(ctx, compute_ctx, cgraph, node_idx);break;}+ if (ctx->num_additional_fused_ops) {+ ggml_vk_unary_mul(ctx, compute_ctx, cgraph, node_idx);+ break;+ }switch (ggml_get_unary_op(node)) {case GGML_UNARY_OP_ELU:@@ -17275,7 +17353,48 @@ static bool ggml_vk_is_empty(ggml_tensor * node) {return ggml_is_empty(node) || node->op == GGML_OP_NONE || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE;}+static bool ggml_vk_can_fuse_unary_mul(const struct ggml_cgraph * cgraph, int unary_idx, int mul_idx) {+ const ggml_tensor * unary = cgraph->nodes[unary_idx];+ const ggml_tensor * mul = cgraph->nodes[mul_idx];++ if (ggml_vk_unary_mul_op_index(ggml_get_unary_op(unary)) < 0) {+ return false;+ }+ if (unary->type != GGML_TYPE_F32 && unary->type != GGML_TYPE_F16) {+ return false;+ }+ if (unary->type != mul->type) {+ return false;+ }+ if (mul->src[0] != unary && mul->src[1] != unary) {+ return false;+ }+ const ggml_tensor * other = (mul->src[0] == unary) ? mul->src[1] : mul->src[0];+ if (other == nullptr || other->type != unary->type) {+ return false;+ }+ if (!ggml_is_contiguous_1(other) || !ggml_is_contiguous_1(unary->src[0])) {+ return false;+ }+ // fastmod needs src to tile into dst+ if (mul->src[0] == unary) {+ return ggml_can_repeat(other, unary);+ }+ return ggml_can_repeat(unary, mul->src[0]);+}++static bool ggml_vk_can_fuse_unary_mul_pair(const struct ggml_cgraph * cgraph, int node_idx) {+ const enum ggml_op ops[] = { GGML_OP_UNARY, GGML_OP_MUL };+ const int outputs[] = { node_idx + 1 };+ return ggml_can_fuse_subgraph(cgraph, node_idx, 2, ops, outputs, 1) &&+ ggml_vk_can_fuse_unary_mul(cgraph, node_idx, node_idx + 1);+}+static bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx, std::initializer_list<enum ggml_op> ops) {+ if (ops.size() == 2 && ops.begin()[0] == GGML_OP_UNARY && ops.begin()[1] == GGML_OP_MUL) {+ return ggml_vk_can_fuse_unary_mul_pair(cgraph, node_idx);+ }+if (!ggml_can_fuse(cgraph, node_idx, ops)) {return false;}@@ -17341,6 +17460,7 @@ static bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct g}}}+auto const &mm_add_ok = [&](const ggml_tensor *mul, const ggml_tensor *add) {const ggml_tensor *bias = add->src[0] == mul ? add->src[1] : add->src[0];@@ -18209,6 +18329,16 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg// they are overwritten, and one workgroup per row. So close enough.op_srcs_fused_elementwise[0] = true;op_srcs_fused_elementwise[1] = true;+ } else if (ggml_vk_can_fuse(ctx, cgraph, i, { GGML_OP_UNARY, GGML_OP_MUL })) {+ ctx->num_additional_fused_ops = 1;+ switch (ggml_get_unary_op(cgraph->nodes[i])) {+ case GGML_UNARY_OP_GELU: fusion_string = "GELU_MUL"; break;+ case GGML_UNARY_OP_SIGMOID: fusion_string = "SIGMOID_MUL"; break;+ case GGML_UNARY_OP_SILU: fusion_string = "SILU_MUL"; break;+ default: fusion_string = "SOFTPLUS_MUL"; break;+ }+ op_srcs_fused_elementwise[0] = true;+ op_srcs_fused_elementwise[1] = true;} else if (ggml_vk_can_fuse_ssm_conv(ctx, cgraph, i, 2)) {ctx->num_additional_fused_ops = 2;fusion_string = "SSM_CONV_BIAS_SILU";@@ -18507,6 +18637,16 @@ static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph *std::set<ggml_tensor *> used_node_set;int first_unused = 0;++ // scheduled or zero-compute nodes in [lo, hi)+ auto const &empty_or_scheduled_between = [&](int lo, int hi) -> bool {+ for (int v = lo; v < hi; ++v) {+ if (!used[v] && !is_empty(graph->nodes[v])) {+ return false;+ }+ }+ return true;+ };while (first_unused < graph->n_nodes) {std::vector<int> current_set;@@ -18622,7 +18762,8 @@ static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph *for (int c = first_unused; c < j; ++c) {if (!used[c] &&is_src_of(graph->nodes[j], graph->nodes[c]) &&- !(j == c+1 && c == current_set.back() && graph->nodes[c]->op == GGML_OP_RMS_NORM && graph->nodes[j]->op == GGML_OP_MUL) &&+ !(c == current_set.back() && graph->nodes[c]->op == GGML_OP_RMS_NORM && graph->nodes[j]->op == GGML_OP_MUL && empty_or_scheduled_between(c+1, j)) &&+ !(c == current_set.back() && graph->nodes[c]->op == GGML_OP_UNARY && graph->nodes[j]->op == GGML_OP_MUL && empty_or_scheduled_between(c+1, j)) &&!(j == c+1 && c == current_set.back() && graph->nodes[c]->op == GGML_OP_MUL_MAT && graph->nodes[j]->op == GGML_OP_ADD) &&!(j == c+1 && c == current_set.back() && graph->nodes[c]->op == GGML_OP_MUL_MAT_ID && graph->nodes[j]->op == GGML_OP_ADD_ID) &&!(j == c+1 && c == current_set.back() && graph->nodes[c]->op == GGML_OP_MUL_MAT_ID && graph->nodes[j]->op == GGML_OP_MUL) &&@@ -18735,6 +18876,27 @@ static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph *}}}+ // UNARY + MUL: pull the consuming MUL forward+ if (j > 0 &&+ graph->nodes[j]->op == GGML_OP_UNARY) {+ for (int k = j + 1; k < std::min(j + 15, graph->n_nodes); ++k) {+ ggml_tensor * mul = graph->nodes[k];+ if (mul->op != GGML_OP_MUL || (mul->src[0] != graph->nodes[j] && mul->src[1] != graph->nodes[j])) {+ continue;+ }+ ggml_tensor * other = (mul->src[0] == graph->nodes[j]) ? mul->src[1] : mul->src[0];+ // the other src must either be weights or already processed+ if (!(other->op == GGML_OP_NONE || used_node_set.find(other) != used_node_set.end())) {+ continue;+ }+ if (!ggml_vk_can_fuse_unary_mul(graph, j, k)) {+ continue;+ }+ current_set.push_back(k);+ used[k] = true;+ break;+ }+ }}}// Second pass grabs view nodes.diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp b/ggml/src/ggml-vulkan/vulkan-shaders/unary.compindex 5ee5275d27..9ee7769bab 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp+++ b/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp@@ -1,9 +1,23 @@#version 450#include "types.glsl"+#if defined(UNARY_MUL_FUSION)+#include "generic_binary_head.glsl"+#else#include "generic_unary_head.glsl"+#endif+#if defined(UNARY_MUL_FUSION)+// OP on src1+layout(constant_id = 1) const bool op_on_b = false;+#endif++#if defined(UNARY_MUL_FUSION)+layout(local_size_x = 256, local_size_y = 1, local_size_z = 1) in;+const uint num_threads = 256;+#elselayout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;+#endiffloat op_abs(float x) {return abs(x);@@ -123,6 +137,7 @@ float op_gelu_erf(float a) {return 0.5f * a * (1.0f + sign_x * y);}+#if !defined(UNARY_MUL_FUSION)float op_xielu(float x) {const float alpha_n = p.param1;const float alpha_p = p.param2;@@ -136,6 +151,7 @@ float op_xielu(float x) {const float min_x_eps = min(x, eps);return (op_expm1(min_x_eps) - x) * alpha_n + beta * x;}+#endiffloat op_floor(float x) {return floor(x);@@ -155,8 +171,28 @@ float op_trunc(float x) {}void main() {- const uint idx = get_idx();-+ uint idx = get_idx();++#if defined(UNARY_MUL_FUSION)+ // keep total threads at 512+ [[unroll]] for (uint iter = 0; iter < 2; ++iter) {+ if (idx >= p.ne) {+ continue;+ }+ uint i00, i01, i02, i03;+ get_indices(idx, i00, i01, i02, i03);++ if (op_on_b) {+ data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] =+ D_TYPE(FLOAT_TYPE(OP(float(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)]))) * FLOAT_TYPE(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]));+ } else {+ data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] =+ D_TYPE(FLOAT_TYPE(OP(float(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]))) * FLOAT_TYPE(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)]));+ }++ idx += num_threads;+ }+#elseif (idx >= p.ne) {return;}@@ -165,4 +201,5 @@ void main() {const uint d_idx = get_doffset() + dst_idx(idx);data_d[d_idx] = D_TYPE(OP(float(data_a[a_idx])));+#endif}diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cppindex 2daafdf438..cb1128dcc6 100644--- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp+++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp@@ -966,6 +966,15 @@ void process_shaders() {string_to_spv("softplus_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_softplus"}});string_to_spv("softplus_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_softplus"}});+ string_to_spv("gelu_mul_f32", "unary.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}, {"OP", "op_gelu"}, {"UNARY_MUL_FUSION", "1"}});+ string_to_spv("gelu_mul_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}, {"OP", "op_gelu"}, {"UNARY_MUL_FUSION", "1"}});+ string_to_spv("sigmoid_mul_f32", "unary.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}, {"OP", "op_sigmoid"}, {"UNARY_MUL_FUSION", "1"}});+ string_to_spv("sigmoid_mul_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}, {"OP", "op_sigmoid"}, {"UNARY_MUL_FUSION", "1"}});+ string_to_spv("silu_mul_f32", "unary.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}, {"OP", "op_silu"}, {"UNARY_MUL_FUSION", "1"}});+ string_to_spv("silu_mul_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}, {"OP", "op_silu"}, {"UNARY_MUL_FUSION", "1"}});+ string_to_spv("softplus_mul_f32","unary.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}, {"OP", "op_softplus"}, {"UNARY_MUL_FUSION", "1"}});+ string_to_spv("softplus_mul_f16","unary.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}, {"OP", "op_softplus"}, {"UNARY_MUL_FUSION", "1"}});+string_to_spv("add1_f16_f16", "add1.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}});string_to_spv("add1_f16_f32", "add1.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}});string_to_spv("add1_f32_f32", "add1.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cppindex 5121578ff6..01e72041bf 100644--- a/tests/test-backend-ops.cpp+++ b/tests/test-backend-ops.cpp@@ -3908,8 +3908,7 @@ struct test_relu_sqr : public test_case {}};-// GGML_OP_UNARY(SILU|SIGMOID|SOFTPLUS) + GGML_OP_MUL (fused operation).-// `layout` and `tail` are used for fallback cases where fusion must be skipped+// GGML_OP_UNARY(GELU|SILU|SIGMOID|SOFTPLUS) + GGML_OP_MUL (fused operation).struct test_unary_mul : public test_case {const ggml_unary_op op;const ggml_type type;@@ -3930,7 +3929,8 @@ struct test_unary_mul : public test_case {// performs; relax the tolerance to match that driftswitch (type) {case GGML_TYPE_F16: return 5e-5;- default: return 1e-7;+ // gelu shader uses exp form, CPU uses tanhf+ default: return op == GGML_UNARY_OP_GELU ? 5e-7 : 1e-7;}}@@ -3989,17 +3989,45 @@ struct test_unary_mul : public test_case {} else if (layout == "bcast") {a = ggml_new_tensor(ctx, type, 4, ne.data());b = ggml_new_tensor_4d(ctx, type, ne[0], 1, 1, 1);+ } else if (layout == "rep_ne0") {+ // repeat on dim 0+ a = ggml_new_tensor(ctx, type, 4, ne.data());+ std::array<int64_t, 4> ne_b = ne;+ ne_b[0] /= 4;+ b = ggml_new_tensor(ctx, type, 4, ne_b.data());+ } else if (layout == "view_mid") {+ // VIEW between UNARY and MUL+ a = ggml_new_tensor(ctx, type, 4, ne.data());+ b = nullptr;+ } else if (layout == "gate") {+ // small gate on src1+ const std::array<int64_t, 4> ne_gate = { 1, ne[1], ne[2], ne[3] };+ a = ggml_new_tensor(ctx, type, 4, ne_gate.data());+ b = ggml_new_tensor(ctx, type, 4, ne.data());} else {GGML_ABORT("unknown layout %s", layout.c_str());}- ggml_set_name(a, "a");- ggml_set_name(b, "b");+ if (a != nullptr) {+ ggml_set_name(a, "a");+ }+ if (b != nullptr) {+ ggml_set_name(b, "b");+ }ggml_tensor * u = ggml_unary(ctx, a, op);ggml_set_name(u, "unary");// a broadcasting operand can only be the second one- const bool second = swap && layout != "bcast";+ const bool second = layout == "gate" || (swap && layout != "bcast" && layout != "view_mid");+ if (layout == "view_mid") {+ std::array<int64_t, 4> ne_base = ne;+ ne_base[0] *= 2;+ ggml_tensor * base = ggml_new_tensor(ctx, type, 4, ne_base.data());+ ggml_set_name(base, "base");+ b = ggml_view_4d(ctx, base, ne[0], ne[1], ne[2], ne[3],+ base->nb[1], base->nb[2], base->nb[3], 0);+ ggml_set_name(b, "b");+ }ggml_tensor * out = second ? ggml_mul(ctx, b, u) : ggml_mul(ctx, u, b);if (tail == "reuse") {@@ -8815,7 +8843,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {}// fused unary + mul (gated activations that are not expressed as GGML_OP_GLU)- for (ggml_unary_op op : { GGML_UNARY_OP_SILU, GGML_UNARY_OP_SIGMOID, GGML_UNARY_OP_SOFTPLUS }) {+ for (ggml_unary_op op : { GGML_UNARY_OP_GELU, GGML_UNARY_OP_SILU, GGML_UNARY_OP_SIGMOID, GGML_UNARY_OP_SOFTPLUS }) {for (ggml_type type : { GGML_TYPE_F16, GGML_TYPE_F32 }) {for (bool swap : { false, true }) {test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, swap));@@ -8826,9 +8854,12 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, true, "pad_other"));test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, true, "halves"));test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, false, "packed", "consumer"));+ test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, false, "bcast"));+ test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, false, "rep_ne0"));+ test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, false, "view_mid"));+ test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, false, "gate"));// must not fusetest_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, false, "strided_dim1"));- test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, false, "bcast"));test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, false, "packed", "reuse"));}}