LLM inference in C/C++
C++ ggml
ggml-cpu : size set_rows conversion buffer for all threads (#30454)
Aman Gupta5cdadfc32616
diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.cindex 5428d5164e..ab82e3d6c0 100644--- a/ggml/src/ggml-cpu/ggml-cpu.c+++ b/ggml/src/ggml-cpu/ggml-cpu.c@@ -2360,11 +2360,11 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {case GGML_OP_MUL_MAT:case GGML_OP_MUL_MAT_ID:case GGML_OP_OUT_PROD:+ case GGML_OP_SET_ROWS:{n_tasks = n_threads;} break;case GGML_OP_GET_ROWS:- case GGML_OP_SET_ROWS:{// FIXME: get_rows can use additional threads, but the cost of launching additional threads// decreases performance with GPU offloading