LLM inference in C/C++
C++ ggml
metal : fix MUL_MAT+ADD fusion when the residual is itself a MUL_MAT (#30100)
Niko Maroulis7e8324f5feba
diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cppindex 2a14cd0c2d..597d5300c0 100644--- a/ggml/src/ggml-metal/ggml-metal-ops.cpp+++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp@@ -2476,7 +2476,8 @@ static int ggml_metal_op_mul_mat_mma(ggml_metal_op_t ctx, int idx) {if (fuse_add) {dst = ctx->node(idx + n_fuse - 1);- res = dst->src[0]->op == GGML_OP_MUL_MAT ? dst->src[1] : dst->src[0];+ // the residual is the other operand of the ADD, by identity: it can itself be a MUL_MAT output+ res = dst->src[0] == op ? dst->src[1] : dst->src[0];}auto pipeline = ggml_metal_library_get_pipeline_mul_mv_mma_auto(lib, op, fuse_add);diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cppindex b080ab4961..efc1777efd 100644--- a/tests/test-backend-ops.cpp+++ b/tests/test-backend-ops.cpp@@ -7233,6 +7233,7 @@ enum mul_mat_add_mode {MUL_MAT_ADD_ROW, // mm + a one-row res, which a same-shape fusion must leave aloneMUL_MAT_ADD_RES_INPLACE, // res += mmMUL_MAT_ADD_B_INPLACE, // b += mm: the sum overwrites the mat-mul input (m == k)+ MUL_MAT_ADD_MM_MM, // mm2 + mm: the residual is itself a mat-mul output, so both operands are MUL_MAT};static std::string var_to_str(mul_mat_add_mode mode) {@@ -7242,6 +7243,7 @@ static std::string var_to_str(mul_mat_add_mode mode) {case MUL_MAT_ADD_ROW: return "mm+row";case MUL_MAT_ADD_RES_INPLACE: return "res+=mm";case MUL_MAT_ADD_B_INPLACE: return "b+=mm";+ case MUL_MAT_ADD_MM_MM: return "mm2+mm";}return "unknown";}@@ -7287,6 +7289,12 @@ struct test_mul_mat_add : public test_case {case MUL_MAT_ADD_RES_MM: out = ggml_add(ctx, res, mm); break;case MUL_MAT_ADD_RES_INPLACE: out = ggml_add_inplace(ctx, res, mm); break;case MUL_MAT_ADD_B_INPLACE: out = ggml_add_inplace(ctx, b, mm); break;+ case MUL_MAT_ADD_MM_MM:+ {+ ggml_tensor * a2 = ggml_new_tensor_2d(ctx, type_a, k, m);+ ggml_tensor * mm2 = ggml_mul_mat(ctx, a2, b);+ out = ggml_add(ctx, mm2, mm);+ } break;}ggml_set_name(out, "out");@@ -10409,7 +10417,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {}}for (ggml_type type_a : {GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, GGML_TYPE_Q5_K, GGML_TYPE_F16}) {- for (mul_mat_add_mode mode : {MUL_MAT_ADD_MM_RES, MUL_MAT_ADD_RES_MM, MUL_MAT_ADD_ROW, MUL_MAT_ADD_RES_INPLACE}) {+ for (mul_mat_add_mode mode : {MUL_MAT_ADD_MM_RES, MUL_MAT_ADD_RES_MM, MUL_MAT_ADD_ROW, MUL_MAT_ADD_RES_INPLACE, MUL_MAT_ADD_MM_MM}) {test_cases.emplace_back(new test_mul_mat_add(type_a, 1000, n, 1024, mode));}test_cases.emplace_back(new test_mul_mat_add(type_a, 2048, n, 2048, MUL_MAT_ADD_B_INPLACE));