LLM inference in C/C++
C++ ggml
CUDA: only support F32/F16 for GGML_OP_REPEAT (#24533)
leonardHONG3686e9d6433f
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuindex 61041bdc16..01c29a8c68 100644--- a/ggml/src/ggml-cuda/ggml-cuda.cu+++ b/ggml/src/ggml-cuda/ggml-cuda.cu@@ -5337,8 +5337,9 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g} break;case GGML_OP_REPEAT:{+ // the CUDA REPEAT path only implements F32/F16; other types assert at runtimeggml_type src0_type = op->src[0]->type;- return src0_type != GGML_TYPE_I32 && src0_type != GGML_TYPE_I16;+ return src0_type == GGML_TYPE_F32 || src0_type == GGML_TYPE_F16;} break;case GGML_OP_REPEAT_BACK:return op->type == GGML_TYPE_F32 && (op->src[0]->ne[2]*op->src[0]->ne[3]) <= (1 << 15);diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cppindex 00d90fc2b4..3afd5878a3 100644--- a/tests/test-backend-ops.cpp+++ b/tests/test-backend-ops.cpp@@ -8086,6 +8086,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {test_cases.emplace_back(new test_repeat(GGML_TYPE_F32, {10, 5, 4, ne3}, {1, 1, 1, 2}));test_cases.emplace_back(new test_repeat(GGML_TYPE_I32, {10, 5, 4, ne3}, {2, 1, 1, 1}));test_cases.emplace_back(new test_repeat(GGML_TYPE_I16, {10, 5, 4, ne3}, {1, 1, 1, 2}));+ test_cases.emplace_back(new test_repeat(GGML_TYPE_BF16, {10, 5, 4, ne3}, {2, 1, 1, 1}));}for (bool view : {false, true}) {