LLM inference in C/C++
C++ ggml
cuda : enable i16 and i32 for DUP (#28897)
Aman Karki4c9233c034fc
diff --git a/docs/ops.md b/docs/ops.mdindex cc8d253820..ceb5d46cf8 100644--- a/docs/ops.md+++ b/docs/ops.md@@ -44,7 +44,7 @@ Legend:| DSV4_HC_COMB | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ || DSV4_HC_POST | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ || DSV4_HC_PRE | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |-| DUP | ❌ | ✅ | ✅ | 🟡 | ❌ | ❌ | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ | ❌ |+| DUP | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ | ❌ || ELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ || EXP | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ || EXPM1 | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |diff --git a/docs/ops/CUDA.csv b/docs/ops/CUDA.csvindex 22c84dd143..570bdf75ee 100644--- a/docs/ops/CUDA.csv+++ b/docs/ops/CUDA.csv@@ -5000,14 +5000,14 @@"CUDA0","REPEAT_BACK","type=f32,ne=[8,6,4,2],nr=[1,1,1,2],v=1","support","1","yes","CUDA""CUDA0","DUP","type=f32,ne=[10,10,20,1]","support","1","yes","CUDA""CUDA0","DUP","type=f16,ne=[10,10,20,1]","support","1","yes","CUDA"-"CUDA0","DUP","type=i32,ne=[10,10,20,1]","support","0","no","CUDA"-"CUDA0","DUP","type=i16,ne=[10,10,20,1]","support","0","no","CUDA"+"CUDA0","DUP","type=i32,ne=[10,10,20,1]","support","1","yes","CUDA"+"CUDA0","DUP","type=i16,ne=[10,10,20,1]","support","1","yes","CUDA""CUDA0","DUP","type=f32,ne=[10,10,5,1],permute=[0,2,1,3]","support","1","yes","CUDA""CUDA0","DUP","type=f16,ne=[10,10,5,1],permute=[0,2,1,3]","support","1","yes","CUDA""CUDA0","DUP","type=f32,ne=[10,10,5,1],permute=[1,0,2,3]","support","1","yes","CUDA""CUDA0","DUP","type=f16,ne=[10,10,5,1],permute=[1,0,2,3]","support","1","yes","CUDA"-"CUDA0","DUP","type=i16,ne=[10,8,3,1],permute=[0,2,1,3]","support","0","no","CUDA"-"CUDA0","DUP","type=i16,ne=[10,8,3,1],permute=[1,2,0,3]","support","0","no","CUDA"+"CUDA0","DUP","type=i16,ne=[10,8,3,1],permute=[0,2,1,3]","support","1","yes","CUDA"+"CUDA0","DUP","type=i16,ne=[10,8,3,1],permute=[1,2,0,3]","support","1","yes","CUDA""CUDA0","SET","type_src=f32,type_dst=f32,ne=[6,5,4,3],dim=1","support","1","yes","CUDA""CUDA0","SET","type_src=f32,type_dst=f32,ne=[6,5,4,3],dim=2","support","1","yes","CUDA""CUDA0","SET","type_src=f32,type_dst=f32,ne=[6,5,4,3],dim=3","support","1","yes","CUDA"diff --git a/ggml/src/ggml-cuda/cpy.cu b/ggml/src/ggml-cuda/cpy.cuindex fd7ffc0bc5..7a99845854 100644--- a/ggml/src/ggml-cuda/cpy.cu+++ b/ggml/src/ggml-cuda/cpy.cu@@ -589,6 +589,14 @@ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, ggggml_cpy_scalar_cuda<int32_t, int32_t>(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);}+ } else if (src0->type == GGML_TYPE_I16 && src1->type == GGML_TYPE_I16) {+ if (can_be_transposed) {+ ggml_cpy_scalar_cuda<int16_t, int16_t, true>+ (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);+ } else {+ ggml_cpy_scalar_cuda<int16_t, int16_t>+ (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);+ }} else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_I32) {if (contiguous_srcs) {ggml_cpy_scalar_contiguous_cuda<float, int32_t>diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuindex 790553888a..43003245c5 100644--- a/ggml/src/ggml-cuda/ggml-cuda.cu+++ b/ggml/src/ggml-cuda/ggml-cuda.cu@@ -5298,10 +5298,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const greturn false;} break;case GGML_OP_DUP:- {- ggml_type src0_type = op->src[0]->type;- return src0_type != GGML_TYPE_I32 && src0_type != GGML_TYPE_I16;- } break;+ return true;case GGML_OP_ARGMAX:case GGML_OP_COUNT_EQUAL:{