Skip to content

LLM inference in C/C++

C++ ggml

vulkan : fuse UNARY(GELU|SIGMOID|SILU|SOFTPLUS) + MUL (#27220)

Ankit Khandelwal64e9bceb2c3a
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
index 738e7cd922..8f37f65b8a 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp
+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
@@ -1071,6 +1071,9 @@ struct vk_device_struct {
vk_pipeline pipeline_trunc[2];
vk_pipeline pipeline_sgn[2];
+ // fused UNARY+MUL pipelines: [op][f16][norepeat][op_on_b]
+ vk_pipeline pipeline_unary_mul[4][2][2][2];
+
vk_pipeline pipeline_add1_f16_f16;
vk_pipeline pipeline_add1_f16_f32;
vk_pipeline pipeline_add1_f32_f32;
@@ -5930,6 +5933,26 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
CREATE_UNARY(expm1)
#undef CREATE_UNARY
+// spec constants: {norepeat, op_on_b}
+#define CREATE_UNARY_MUL(name, idx) \
+ for (int dt = 0; dt < 2; ++dt) { \
+ const size_t len_ = dt ? name ## _mul_f16_len : name ## _mul_f32_len; \
+ const unsigned char * data_ = dt ? name ## _mul_f16_data : name ## _mul_f32_data; \
+ const std::string dts_ = dt ? "f16" : "f32"; \
+ for (int ob = 0; ob < 2; ++ob) \
+ for (int nr = 0; nr < 2; ++nr) \
+ ggml_vk_create_pipeline(device, device->pipeline_unary_mul[(idx)][dt][nr][ob], \
+ (#name "_mul" + std::string(ob ? "_b" : "") + "_" + dts_ + (nr ? "_norepeat" : "")).c_str(), \
+ len_, data_, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, \
+ { (uint32_t) nr, (uint32_t) ob }, 1); \
+ }
+
+ CREATE_UNARY_MUL(gelu, 0)
+ CREATE_UNARY_MUL(sigmoid, 1)
+ CREATE_UNARY_MUL(silu, 2)
+ CREATE_UNARY_MUL(softplus, 3)
+#undef CREATE_UNARY_MUL
+
ggml_vk_create_pipeline(device, device->pipeline_add1_f16_f16, "add1_f16_f16", add1_f16_f16_len, add1_f16_f16_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_add1_f16_f32, "add1_f16_f32", add1_f16_f32_len, add1_f16_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_add1_f32_f32, "add1_f32_f32", add1_f32_f32_len, add1_f32_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {512, 1, 1}, {}, 1);
@@ -12416,7 +12439,7 @@ template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk
}
template<typename PC>
-static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst, ggml_op op, PC&& pc) {
+static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst, ggml_op op, PC&& pc, vk_pipeline pipeline_override = nullptr) {
VK_LOG_DEBUG("ggml_vk_op_f32((" << src0 << ", name=" << src0->name << ", type=" << src0->type << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3];
if (src1 != nullptr) {
std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << src1->type << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3];
@@ -12447,7 +12470,12 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co
init_pushconst_fastdiv(pc);
- vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, src0, src1, src2, dst, op);
+ vk_pipeline pipeline;
+ if (pipeline_override) {
+ pipeline = pipeline_override;
+ } else {
+ pipeline = ggml_vk_op_get_pipeline(ctx, src0, src1, src2, dst, op);
+ }
if (pipeline == nullptr) {
std::cerr << "ggml_vulkan: Error: Missing op: " << ggml_op_name(op) << " for " << ggml_type_name(src0->type);
@@ -13032,6 +13060,52 @@ static void ggml_vk_mul(ggml_backend_vk_context * ctx, vk_context& subctx, const
});
}
+// index into device->pipeline_unary_mul for the supported unary ops, or -1
+static int ggml_vk_unary_mul_op_index(ggml_unary_op op) {
+ switch (op) {
+ case GGML_UNARY_OP_GELU: return 0;
+ case GGML_UNARY_OP_SIGMOID: return 1;
+ case GGML_UNARY_OP_SILU: return 2;
+ case GGML_UNARY_OP_SOFTPLUS: return 3;
+ default: return -1;
+ }
+}
+
+static void ggml_vk_unary_mul(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) {
+ const ggml_tensor * unary = cgraph->nodes[node_idx];
+ ggml_tensor * mul = cgraph->nodes[node_idx + 1];
+
+ // unary on src1 that tiles into src0
+ const bool op_on_b = mul->src[1] == unary &&
+ !ggml_are_same_shape(unary->src[0], mul->src[0]) &&
+ ggml_can_repeat(unary, mul->src[0]);
+
+ const ggml_tensor * src0 = op_on_b ? mul->src[0] : unary->src[0];
+ const ggml_tensor * src1 = op_on_b ? unary->src[0] :
+ ((mul->src[0] == unary) ? mul->src[1] : mul->src[0]);
+
+ const bool f16 = src0->type == GGML_TYPE_F16;
+ const bool norepeat = ggml_are_same_shape(src0, src1);
+ const int oi = ggml_vk_unary_mul_op_index(ggml_get_unary_op(unary));
+ if (oi < 0) {
+ GGML_ABORT("fatal error");
+ }
+ vk_pipeline pipeline = ctx->device->pipeline_unary_mul[oi][f16][norepeat][op_on_b];
+
+ const uint32_t src0_type_size = ggml_type_size(src0->type);
+ const uint32_t src1_type_size = ggml_type_size(src1->type);
+ const uint32_t dst_type_size = ggml_type_size(mul->type);
+
+ ggml_vk_op_f32<vk_op_binary_push_constants>(ctx, subctx, src0, src1, nullptr, nullptr, mul, GGML_OP_UNARY, {
+ (uint32_t)ggml_nelements(op_on_b ? mul : src0),
+ (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size,
+ (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size,
+ (uint32_t) mul->ne[0], (uint32_t) mul->ne[1], (uint32_t) mul->ne[2],(uint32_t) mul->ne[3], (uint32_t) mul->nb[0] / dst_type_size, (uint32_t) mul->nb[1] / dst_type_size, (uint32_t) mul->nb[2] / dst_type_size, (uint32_t) mul->nb[3] / dst_type_size,
+ 0,
+ 0.0f, 0.0f, 0,
+ }, pipeline);
+}
+
static void ggml_vk_div(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
const uint32_t src0_type_size = ggml_type_size(src0->type);
const uint32_t src1_type_size = ggml_type_size(src1->type);
@@ -16314,6 +16388,10 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgr
ggml_vk_topk_moe(ctx, compute_ctx, cgraph, node_idx);
break;
}
+ if (ctx->num_additional_fused_ops) {
+ ggml_vk_unary_mul(ctx, compute_ctx, cgraph, node_idx);
+ break;
+ }
switch (ggml_get_unary_op(node)) {
case GGML_UNARY_OP_ELU:
@@ -17275,7 +17353,48 @@ static bool ggml_vk_is_empty(ggml_tensor * node) {
return ggml_is_empty(node) || node->op == GGML_OP_NONE || node->op == GGML_OP_RESHAPE || node->op == GGML_OP_TRANSPOSE || node->op == GGML_OP_VIEW || node->op == GGML_OP_PERMUTE;
}
+static bool ggml_vk_can_fuse_unary_mul(const struct ggml_cgraph * cgraph, int unary_idx, int mul_idx) {
+ const ggml_tensor * unary = cgraph->nodes[unary_idx];
+ const ggml_tensor * mul = cgraph->nodes[mul_idx];
+
+ if (ggml_vk_unary_mul_op_index(ggml_get_unary_op(unary)) < 0) {
+ return false;
+ }
+ if (unary->type != GGML_TYPE_F32 && unary->type != GGML_TYPE_F16) {
+ return false;
+ }
+ if (unary->type != mul->type) {
+ return false;
+ }
+ if (mul->src[0] != unary && mul->src[1] != unary) {
+ return false;
+ }
+ const ggml_tensor * other = (mul->src[0] == unary) ? mul->src[1] : mul->src[0];
+ if (other == nullptr || other->type != unary->type) {
+ return false;
+ }
+ if (!ggml_is_contiguous_1(other) || !ggml_is_contiguous_1(unary->src[0])) {
+ return false;
+ }
+ // fastmod needs src to tile into dst
+ if (mul->src[0] == unary) {
+ return ggml_can_repeat(other, unary);
+ }
+ return ggml_can_repeat(unary, mul->src[0]);
+}
+
+static bool ggml_vk_can_fuse_unary_mul_pair(const struct ggml_cgraph * cgraph, int node_idx) {
+ const enum ggml_op ops[] = { GGML_OP_UNARY, GGML_OP_MUL };
+ const int outputs[] = { node_idx + 1 };
+ return ggml_can_fuse_subgraph(cgraph, node_idx, 2, ops, outputs, 1) &&
+ ggml_vk_can_fuse_unary_mul(cgraph, node_idx, node_idx + 1);
+}
+
static bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx, std::initializer_list<enum ggml_op> ops) {
+ if (ops.size() == 2 && ops.begin()[0] == GGML_OP_UNARY && ops.begin()[1] == GGML_OP_MUL) {
+ return ggml_vk_can_fuse_unary_mul_pair(cgraph, node_idx);
+ }
+
if (!ggml_can_fuse(cgraph, node_idx, ops)) {
return false;
}
@@ -17341,6 +17460,7 @@ static bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct g
}
}
}
+
auto const &mm_add_ok = [&](const ggml_tensor *mul, const ggml_tensor *add) {
const ggml_tensor *bias = add->src[0] == mul ? add->src[1] : add->src[0];
@@ -18209,6 +18329,16 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg
// they are overwritten, and one workgroup per row. So close enough.
op_srcs_fused_elementwise[0] = true;
op_srcs_fused_elementwise[1] = true;
+ } else if (ggml_vk_can_fuse(ctx, cgraph, i, { GGML_OP_UNARY, GGML_OP_MUL })) {
+ ctx->num_additional_fused_ops = 1;
+ switch (ggml_get_unary_op(cgraph->nodes[i])) {
+ case GGML_UNARY_OP_GELU: fusion_string = "GELU_MUL"; break;
+ case GGML_UNARY_OP_SIGMOID: fusion_string = "SIGMOID_MUL"; break;
+ case GGML_UNARY_OP_SILU: fusion_string = "SILU_MUL"; break;
+ default: fusion_string = "SOFTPLUS_MUL"; break;
+ }
+ op_srcs_fused_elementwise[0] = true;
+ op_srcs_fused_elementwise[1] = true;
} else if (ggml_vk_can_fuse_ssm_conv(ctx, cgraph, i, 2)) {
ctx->num_additional_fused_ops = 2;
fusion_string = "SSM_CONV_BIAS_SILU";
@@ -18507,6 +18637,16 @@ static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph *
std::set<ggml_tensor *> used_node_set;
int first_unused = 0;
+
+ // scheduled or zero-compute nodes in [lo, hi)
+ auto const &empty_or_scheduled_between = [&](int lo, int hi) -> bool {
+ for (int v = lo; v < hi; ++v) {
+ if (!used[v] && !is_empty(graph->nodes[v])) {
+ return false;
+ }
+ }
+ return true;
+ };
while (first_unused < graph->n_nodes) {
std::vector<int> current_set;
@@ -18622,7 +18762,8 @@ static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph *
for (int c = first_unused; c < j; ++c) {
if (!used[c] &&
is_src_of(graph->nodes[j], graph->nodes[c]) &&
- !(j == c+1 && c == current_set.back() && graph->nodes[c]->op == GGML_OP_RMS_NORM && graph->nodes[j]->op == GGML_OP_MUL) &&
+ !(c == current_set.back() && graph->nodes[c]->op == GGML_OP_RMS_NORM && graph->nodes[j]->op == GGML_OP_MUL && empty_or_scheduled_between(c+1, j)) &&
+ !(c == current_set.back() && graph->nodes[c]->op == GGML_OP_UNARY && graph->nodes[j]->op == GGML_OP_MUL && empty_or_scheduled_between(c+1, j)) &&
!(j == c+1 && c == current_set.back() && graph->nodes[c]->op == GGML_OP_MUL_MAT && graph->nodes[j]->op == GGML_OP_ADD) &&
!(j == c+1 && c == current_set.back() && graph->nodes[c]->op == GGML_OP_MUL_MAT_ID && graph->nodes[j]->op == GGML_OP_ADD_ID) &&
!(j == c+1 && c == current_set.back() && graph->nodes[c]->op == GGML_OP_MUL_MAT_ID && graph->nodes[j]->op == GGML_OP_MUL) &&
@@ -18735,6 +18876,27 @@ static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph *
}
}
}
+ // UNARY + MUL: pull the consuming MUL forward
+ if (j > 0 &&
+ graph->nodes[j]->op == GGML_OP_UNARY) {
+ for (int k = j + 1; k < std::min(j + 15, graph->n_nodes); ++k) {
+ ggml_tensor * mul = graph->nodes[k];
+ if (mul->op != GGML_OP_MUL || (mul->src[0] != graph->nodes[j] && mul->src[1] != graph->nodes[j])) {
+ continue;
+ }
+ ggml_tensor * other = (mul->src[0] == graph->nodes[j]) ? mul->src[1] : mul->src[0];
+ // the other src must either be weights or already processed
+ if (!(other->op == GGML_OP_NONE || used_node_set.find(other) != used_node_set.end())) {
+ continue;
+ }
+ if (!ggml_vk_can_fuse_unary_mul(graph, j, k)) {
+ continue;
+ }
+ current_set.push_back(k);
+ used[k] = true;
+ break;
+ }
+ }
}
}
// Second pass grabs view nodes.
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp b/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp
index 5ee5275d27..9ee7769bab 100644
--- a/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp
@@ -1,9 +1,23 @@
#version 450
#include "types.glsl"
+#if defined(UNARY_MUL_FUSION)
+#include "generic_binary_head.glsl"
+#else
#include "generic_unary_head.glsl"
+#endif
+#if defined(UNARY_MUL_FUSION)
+// OP on src1
+layout(constant_id = 1) const bool op_on_b = false;
+#endif
+
+#if defined(UNARY_MUL_FUSION)
+layout(local_size_x = 256, local_size_y = 1, local_size_z = 1) in;
+const uint num_threads = 256;
+#else
layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
+#endif
float op_abs(float x) {
return abs(x);
@@ -123,6 +137,7 @@ float op_gelu_erf(float a) {
return 0.5f * a * (1.0f + sign_x * y);
}
+#if !defined(UNARY_MUL_FUSION)
float op_xielu(float x) {
const float alpha_n = p.param1;
const float alpha_p = p.param2;
@@ -136,6 +151,7 @@ float op_xielu(float x) {
const float min_x_eps = min(x, eps);
return (op_expm1(min_x_eps) - x) * alpha_n + beta * x;
}
+#endif
float op_floor(float x) {
return floor(x);
@@ -155,8 +171,28 @@ float op_trunc(float x) {
}
void main() {
- const uint idx = get_idx();
-
+ uint idx = get_idx();
+
+#if defined(UNARY_MUL_FUSION)
+ // keep total threads at 512
+ [[unroll]] for (uint iter = 0; iter < 2; ++iter) {
+ if (idx >= p.ne) {
+ continue;
+ }
+ uint i00, i01, i02, i03;
+ get_indices(idx, i00, i01, i02, i03);
+
+ if (op_on_b) {
+ data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] =
+ D_TYPE(FLOAT_TYPE(OP(float(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)]))) * FLOAT_TYPE(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]));
+ } else {
+ data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] =
+ D_TYPE(FLOAT_TYPE(OP(float(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]))) * FLOAT_TYPE(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)]));
+ }
+
+ idx += num_threads;
+ }
+#else
if (idx >= p.ne) {
return;
}
@@ -165,4 +201,5 @@ void main() {
const uint d_idx = get_doffset() + dst_idx(idx);
data_d[d_idx] = D_TYPE(OP(float(data_a[a_idx])));
+#endif
}
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp
index 2daafdf438..cb1128dcc6 100644
--- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp
@@ -966,6 +966,15 @@ void process_shaders() {
string_to_spv("softplus_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_softplus"}});
string_to_spv("softplus_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_softplus"}});
+ string_to_spv("gelu_mul_f32", "unary.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}, {"OP", "op_gelu"}, {"UNARY_MUL_FUSION", "1"}});
+ string_to_spv("gelu_mul_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}, {"OP", "op_gelu"}, {"UNARY_MUL_FUSION", "1"}});
+ string_to_spv("sigmoid_mul_f32", "unary.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}, {"OP", "op_sigmoid"}, {"UNARY_MUL_FUSION", "1"}});
+ string_to_spv("sigmoid_mul_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}, {"OP", "op_sigmoid"}, {"UNARY_MUL_FUSION", "1"}});
+ string_to_spv("silu_mul_f32", "unary.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}, {"OP", "op_silu"}, {"UNARY_MUL_FUSION", "1"}});
+ string_to_spv("silu_mul_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}, {"OP", "op_silu"}, {"UNARY_MUL_FUSION", "1"}});
+ string_to_spv("softplus_mul_f32","unary.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}, {"OP", "op_softplus"}, {"UNARY_MUL_FUSION", "1"}});
+ string_to_spv("softplus_mul_f16","unary.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}, {"OP", "op_softplus"}, {"UNARY_MUL_FUSION", "1"}});
+
string_to_spv("add1_f16_f16", "add1.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}});
string_to_spv("add1_f16_f32", "add1.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}});
string_to_spv("add1_f32_f32", "add1.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index 5121578ff6..01e72041bf 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -3908,8 +3908,7 @@ struct test_relu_sqr : public test_case {
}
};
-// GGML_OP_UNARY(SILU|SIGMOID|SOFTPLUS) + GGML_OP_MUL (fused operation).
-// `layout` and `tail` are used for fallback cases where fusion must be skipped
+// GGML_OP_UNARY(GELU|SILU|SIGMOID|SOFTPLUS) + GGML_OP_MUL (fused operation).
struct test_unary_mul : public test_case {
const ggml_unary_op op;
const ggml_type type;
@@ -3930,7 +3929,8 @@ struct test_unary_mul : public test_case {
// performs; relax the tolerance to match that drift
switch (type) {
case GGML_TYPE_F16: return 5e-5;
- default: return 1e-7;
+ // gelu shader uses exp form, CPU uses tanhf
+ default: return op == GGML_UNARY_OP_GELU ? 5e-7 : 1e-7;
}
}
@@ -3989,17 +3989,45 @@ struct test_unary_mul : public test_case {
} else if (layout == "bcast") {
a = ggml_new_tensor(ctx, type, 4, ne.data());
b = ggml_new_tensor_4d(ctx, type, ne[0], 1, 1, 1);
+ } else if (layout == "rep_ne0") {
+ // repeat on dim 0
+ a = ggml_new_tensor(ctx, type, 4, ne.data());
+ std::array<int64_t, 4> ne_b = ne;
+ ne_b[0] /= 4;
+ b = ggml_new_tensor(ctx, type, 4, ne_b.data());
+ } else if (layout == "view_mid") {
+ // VIEW between UNARY and MUL
+ a = ggml_new_tensor(ctx, type, 4, ne.data());
+ b = nullptr;
+ } else if (layout == "gate") {
+ // small gate on src1
+ const std::array<int64_t, 4> ne_gate = { 1, ne[1], ne[2], ne[3] };
+ a = ggml_new_tensor(ctx, type, 4, ne_gate.data());
+ b = ggml_new_tensor(ctx, type, 4, ne.data());
} else {
GGML_ABORT("unknown layout %s", layout.c_str());
}
- ggml_set_name(a, "a");
- ggml_set_name(b, "b");
+ if (a != nullptr) {
+ ggml_set_name(a, "a");
+ }
+ if (b != nullptr) {
+ ggml_set_name(b, "b");
+ }
ggml_tensor * u = ggml_unary(ctx, a, op);
ggml_set_name(u, "unary");
// a broadcasting operand can only be the second one
- const bool second = swap && layout != "bcast";
+ const bool second = layout == "gate" || (swap && layout != "bcast" && layout != "view_mid");
+ if (layout == "view_mid") {
+ std::array<int64_t, 4> ne_base = ne;
+ ne_base[0] *= 2;
+ ggml_tensor * base = ggml_new_tensor(ctx, type, 4, ne_base.data());
+ ggml_set_name(base, "base");
+ b = ggml_view_4d(ctx, base, ne[0], ne[1], ne[2], ne[3],
+ base->nb[1], base->nb[2], base->nb[3], 0);
+ ggml_set_name(b, "b");
+ }
ggml_tensor * out = second ? ggml_mul(ctx, b, u) : ggml_mul(ctx, u, b);
if (tail == "reuse") {
@@ -8815,7 +8843,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
}
// fused unary + mul (gated activations that are not expressed as GGML_OP_GLU)
- for (ggml_unary_op op : { GGML_UNARY_OP_SILU, GGML_UNARY_OP_SIGMOID, GGML_UNARY_OP_SOFTPLUS }) {
+ for (ggml_unary_op op : { GGML_UNARY_OP_GELU, GGML_UNARY_OP_SILU, GGML_UNARY_OP_SIGMOID, GGML_UNARY_OP_SOFTPLUS }) {
for (ggml_type type : { GGML_TYPE_F16, GGML_TYPE_F32 }) {
for (bool swap : { false, true }) {
test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, swap));
@@ -8826,9 +8854,12 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, true, "pad_other"));
test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, true, "halves"));
test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, false, "packed", "consumer"));
+ test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, false, "bcast"));
+ test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, false, "rep_ne0"));
+ test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, false, "view_mid"));
+ test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, false, "gate"));
// must not fuse
test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, false, "strided_dim1"));
- test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, false, "bcast"));
test_cases.emplace_back(new test_unary_mul(op, type, { 128, 2, 2, 2 }, false, "packed", "reuse"));
}
}