LLM inference in C/C++
C++ ggml
cuda : add BF16 support for XIELU (#29955)
qiao_px5ad1c5da0ad7
diff --git a/docs/ops.md b/docs/ops.mdindex d47dda94ad..8a594fa4aa 100644--- a/docs/ops.md+++ b/docs/ops.md@@ -129,4 +129,4 @@ Legend:| TRI | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ || TRUNC | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ || UPSCALE | ❌ | 🟡 | ✅ | ✅ | ❌ | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |-| XIELU | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |+| XIELU | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |diff --git a/docs/ops/CUDA.csv b/docs/ops/CUDA.csvindex 598aee9575..7499a49276 100644--- a/docs/ops/CUDA.csv+++ b/docs/ops/CUDA.csv@@ -9934,7 +9934,12 @@"CUDA0","CUMSUM","type=f32,ne=[2048,5,4,3]","support","1","yes","CUDA""CUDA0","CUMSUM","type=f32,ne=[242004,1,1,1]","support","1","yes","CUDA""CUDA0","CUMSUM","type=f32,ne=[375960,1,1,1]","support","1","yes","CUDA"-"CUDA0","XIELU","type=f32,ne=[10,5,4,3]","support","0","no","CUDA"+"CUDA0","XIELU","type=f32,ne=[10,5,4,3]","support","1","yes","CUDA"+"CUDA0","XIELU","type=f16,ne=[10,5,4,3]","support","1","yes","CUDA"+"CUDA0","XIELU","type=bf16,ne=[10,5,4,3]","support","1","yes","CUDA"+"CUDA0","XIELU","type=f32,ne=[512,16,1,1]","support","1","yes","CUDA"+"CUDA0","XIELU","type=f16,ne=[512,16,1,1]","support","1","yes","CUDA"+"CUDA0","XIELU","type=bf16,ne=[512,16,1,1]","support","1","yes","CUDA""CUDA0","TRI","type=f32,ne=[10,10,4,3],tri_type=3","support","1","yes","CUDA""CUDA0","TRI","type=f32,ne=[10,10,4,3],tri_type=2","support","1","yes","CUDA""CUDA0","TRI","type=f32,ne=[10,10,4,3],tri_type=1","support","1","yes","CUDA"diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuindex 0f88dfe880..33ebfdfa05 100644--- a/ggml/src/ggml-cuda/ggml-cuda.cu+++ b/ggml/src/ggml-cuda/ggml-cuda.cu@@ -5314,9 +5314,6 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const gcase GGML_UNARY_OP_CEIL:case GGML_UNARY_OP_ROUND:case GGML_UNARY_OP_TRUNC:- if (op->src[0]->type == GGML_TYPE_BF16 && ggml_get_unary_op(op) == GGML_UNARY_OP_XIELU) {- return false;- }// TODO: should become://return ggml_is_contiguous_rows(op->src[0]);return ggml_is_contiguous(op->src[0]);diff --git a/ggml/src/ggml-cuda/unary.cu b/ggml/src/ggml-cuda/unary.cuindex 84788c5dc5..c9a8cf6b00 100644--- a/ggml/src/ggml-cuda/unary.cu+++ b/ggml/src/ggml-cuda/unary.cu@@ -547,8 +547,8 @@ void ggml_cuda_op_xielu(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {GGML_ASSERT(ggml_is_contiguous(src0));- GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);- GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);+ GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16);+ GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16 || dst->type == GGML_TYPE_BF16);GGML_ASSERT(src0->type == dst->type);const float alpha_n = ggml_get_op_params_f32(dst, 1);@@ -558,6 +558,8 @@ void ggml_cuda_op_xielu(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {if (src0->type == GGML_TYPE_F16) {xielu_cuda((const half *)src0_d, (half *)dst_d, ggml_nelements(src0), alpha_n, alpha_p, beta, eps, stream);+ } else if (src0->type == GGML_TYPE_BF16) {+ xielu_cuda((const nv_bfloat16 *)src0_d, (nv_bfloat16 *)dst_d, ggml_nelements(src0), alpha_n, alpha_p, beta, eps, stream);} else {xielu_cuda((const float *)src0_d, (float *)dst_d, ggml_nelements(src0), alpha_n, alpha_p, beta, eps, stream);}diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cppindex cbe59178d3..b080ab4961 100644--- a/tests/test-backend-ops.cpp+++ b/tests/test-backend-ops.cpp@@ -11189,8 +11189,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {test_cases.emplace_back(new test_xielu());test_cases.emplace_back(new test_xielu(GGML_TYPE_F16));+ test_cases.emplace_back(new test_xielu(GGML_TYPE_BF16));test_cases.emplace_back(new test_xielu(GGML_TYPE_F32, { 512, 16, 1, 1 }));test_cases.emplace_back(new test_xielu(GGML_TYPE_F16, { 512, 16, 1, 1 }));+ test_cases.emplace_back(new test_xielu(GGML_TYPE_BF16, { 512, 16, 1, 1 }));test_cases.emplace_back(new test_tri(GGML_TRI_TYPE_LOWER));test_cases.emplace_back(new test_tri(GGML_TRI_TYPE_LOWER_DIAG));