LLM inference in C/C++
C++ ggml
ggml : add note about mul_mat_id + disable mul_mat_id_dup tests (#30501)
Georgi Gerganov8c2134a8755d
diff --git a/ggml/include/ggml.h b/ggml/include/ggml.hindex 224bdef927..23ea6437e2 100644--- a/ggml/include/ggml.h+++ b/ggml/include/ggml.h@@ -1497,6 +1497,11 @@ extern "C" {enum ggml_op_hint hint);// indirect matrix multiplication+ //+ // [TAG_MUL_MAT_ID_DUPLICATE_ID]+ // note: this op has undefined behaviour if a given `a` is referenced more than once for a given `b`+ // (i.e. duplicate experts per token are not guaranteed to be supported in all backends)+ // TODO: add backend support: https://github.com/ggml-org/llama.cpp/pull/29998#issuecomment-6153742533GGML_API struct ggml_tensor * ggml_mul_mat_id(struct ggml_context * ctx,struct ggml_tensor * as,diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cppindex 3ad705deea..1e95bf50c5 100644--- a/tests/test-backend-ops.cpp+++ b/tests/test-backend-ops.cpp@@ -11028,6 +11028,8 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_TQ1_0, GGML_TYPE_F32, 28, 10, false, 1024, 1, 4096));test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_TQ1_0, GGML_TYPE_F32, 128, 8, false, 1024, 1, 2048));+ // TODO: disabled until CUDA backend support is added [TAG_MUL_MAT_ID_DUPLICATE_ID]+#if 0// repeated expert ids within a rowfor (ggml_type ta : {GGML_TYPE_F16, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0}) {for (int n : {9, 16, 33, 64}) {@@ -11039,6 +11041,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {test_cases.emplace_back(new test_mul_mat_id_dup(ta, GGML_TYPE_F32, 1025, 10, false, 64, 33, 256, 1.0f, 0, true));test_cases.emplace_back(new test_mul_mat_id_dup(ta, GGML_TYPE_F32, 2050, 10, true, 64, 33, 256));}+#endiffor (ggml_type type_a : all_types) {test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 4, 2, false, 64, 16, 3*ggml_blck_size(type_a)));