LLM inference in C/C++
C++ ggml
common: apply CPU parameters across tools (#27026)
Niklas Wenzela94d563ed801
diff --git a/common/common.cpp b/common/common.cppindex 5639bdf412..cea6d3f5cc 100644--- a/common/common.cpp+++ b/common/common.cpp@@ -1275,6 +1275,8 @@ struct common_init_result::impl {// note: the order in which model, context, etc. are declared matters because their destructors will be called bottom-to-top+ common_threadpools threadpools;+llama_model_ptr model;llama_context_ptr context;@@ -1376,6 +1378,10 @@ common_init_result::common_init_result(common_params & params, bool model_only)}pimpl->context.reset(lctx);++ set_process_priority(params.cpuparams.priority);++ pimpl->threadpools.init(lctx, params);}llama_model * common_init_result::model() {@@ -1724,6 +1730,10 @@ struct llama_context_params common_context_params_to_llama(const common_params &return cparams;}+//+// Threadpool utils+//+struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const common_cpu_params & params) {struct ggml_threadpool_params tpp;@@ -1740,6 +1750,56 @@ struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const commoreturn tpp;}+common_threadpools::~common_threadpools() {+ if (!free_fn) {+ return;+ }+ free_fn(threadpool);+ free_fn(threadpool_batch);+}++void common_threadpools::init(llama_context * ctx, const common_params & params) {+ GGML_ASSERT(!threadpool);+ GGML_ASSERT(!threadpool_batch);++ COM_INF("llama threadpool init, n_threads = %d\n", (int) params.cpuparams.n_threads);++ auto * cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);+ if (!cpu_dev) {+ COM_WRN("%s", "no CPU backend found\n");+ return;+ }+ auto * reg = ggml_backend_dev_backend_reg(cpu_dev);+ auto * ggml_threadpool_new_fn = (decltype(ggml_threadpool_new) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_new");+ free_fn = (decltype(ggml_threadpool_free) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_free");++ struct ggml_threadpool_params tpp_batch =+ ggml_threadpool_params_from_cpu_params(params.cpuparams_batch);+ struct ggml_threadpool_params tpp =+ ggml_threadpool_params_from_cpu_params(params.cpuparams);++ if (!ggml_threadpool_params_match(&tpp, &tpp_batch)) {+ threadpool_batch = ggml_threadpool_new_fn(&tpp_batch);+ if (!threadpool_batch) {+ COM_WRN("batch threadpool create failed : n_threads %d\n", tpp_batch.n_threads);+ return;+ }++ // start the non-batch threadpool in the paused state+ tpp.paused = true;+ }++ threadpool = ggml_threadpool_new_fn(&tpp);+ if (!threadpool) {+ COM_WRN("threadpool create failed : n_threads %d\n", tpp.n_threads);+ free_fn(threadpool_batch);+ threadpool_batch = nullptr;+ return;+ }++ llama_attach_threadpool(ctx, threadpool, threadpool_batch);+}+//// Batch utils//diff --git a/common/common.h b/common/common.hindex b16bd3bb68..d8a16897b8 100644--- a/common/common.h+++ b/common/common.h@@ -929,9 +929,8 @@ using common_init_result_ptr = std::unique_ptr<common_init_result>;common_init_result_ptr common_init_from_params(common_params & params, bool model_only = false);-struct llama_model_params common_model_params_to_llama ( common_params & params);-struct llama_context_params common_context_params_to_llama(const common_params & params);-struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const common_cpu_params & params);+struct llama_model_params common_model_params_to_llama ( common_params & params);+struct llama_context_params common_context_params_to_llama(const common_params & params);// clear LoRA adapters from context, then apply new list of adaptersvoid common_set_adapter_lora(struct llama_context * ctx, std::vector<common_adapter_lora_info> & lora);@@ -942,6 +941,28 @@ std::string common_get_model_endpoint();// for testing purposeschar * common_get_model_or_exit(int, char*[]);+//+// Threadpool utils+//++struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const common_cpu_params & params);++struct common_threadpools {+ common_threadpools() = default;+ ~common_threadpools();++ common_threadpools(const common_threadpools &) = delete;+ common_threadpools & operator=(const common_threadpools &) = delete;++ void init(llama_context * ctx, const common_params & params);++private:+ ggml_threadpool * threadpool = nullptr;+ ggml_threadpool * threadpool_batch = nullptr;++ decltype(ggml_threadpool_free) * free_fn = nullptr;+};+//// Context utils//diff --git a/tools/completion/completion.cpp b/tools/completion/completion.cppindex 6747558fc5..941b7399b2 100644--- a/tools/completion/completion.cpp+++ b/tools/completion/completion.cpp@@ -160,47 +160,6 @@ int llama_completion(int argc, char ** argv) {// start measuring performance timings from herellama_perf_context_reset(ctx);- LOG_INF("%s: llama threadpool init, n_threads = %d\n", __func__, (int) params.cpuparams.n_threads);-- auto * cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);- if (!cpu_dev) {- LOG_ERR("%s: no CPU backend found\n", __func__);- return 1;- }- auto * reg = ggml_backend_dev_backend_reg(cpu_dev);- auto * ggml_threadpool_new_fn = (decltype(ggml_threadpool_new) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_new");- auto * ggml_threadpool_free_fn = (decltype(ggml_threadpool_free) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_free");-- struct ggml_threadpool_params tpp_batch =- ggml_threadpool_params_from_cpu_params(params.cpuparams_batch);- struct ggml_threadpool_params tpp =- ggml_threadpool_params_from_cpu_params(params.cpuparams);-- if (!set_process_priority(params.cpuparams.priority)) {- LOG_ERR("%s: error: failed to set process priority\n", __func__);- return 1;- }-- struct ggml_threadpool * threadpool_batch = NULL;- if (!ggml_threadpool_params_match(&tpp, &tpp_batch)) {- threadpool_batch = ggml_threadpool_new_fn(&tpp_batch);- if (!threadpool_batch) {- LOG_ERR("%s: batch threadpool create failed : n_threads %d\n", __func__, tpp_batch.n_threads);- return 1;- }-- // start the non-batch threadpool in the paused state- tpp.paused = true;- }-- struct ggml_threadpool * threadpool = ggml_threadpool_new_fn(&tpp);- if (!threadpool) {- LOG_ERR("%s: threadpool create failed : n_threads %d\n", __func__, tpp.n_threads);- return 1;- }-- llama_attach_threadpool(ctx, threadpool, threadpool_batch);-const int n_ctx_train = llama_model_n_ctx_train(model);const int n_ctx = llama_n_ctx(ctx);@@ -993,8 +952,5 @@ int llama_completion(int argc, char ** argv) {llama_backend_free();- ggml_threadpool_free_fn(threadpool);- ggml_threadpool_free_fn(threadpool_batch);-return 0;}