LLM inference in C/C++
C++ ggml
sync : ggml (#28379)
Georgi Gerganov64a155d242cb
diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txtindex b75506a26b..d76ed8ab04 100644--- a/ggml/CMakeLists.txt+++ b/ggml/CMakeLists.txt@@ -4,7 +4,7 @@ project("ggml" C CXX ASM)### GGML Versionset(GGML_VERSION_MAJOR 0)-set(GGML_VERSION_MINOR 22)+set(GGML_VERSION_MINOR 23)set(GGML_VERSION_PATCH 0)set(GGML_VERSION_BASE "${GGML_VERSION_MAJOR}.${GGML_VERSION_MINOR}.${GGML_VERSION_PATCH}")diff --git a/ggml/include/ggml-backend.h b/ggml/include/ggml-backend.hindex 27375bd0a5..cc3f8cd36e 100644--- a/ggml/include/ggml-backend.h+++ b/ggml/include/ggml-backend.h@@ -424,10 +424,6 @@ extern "C" {// Compare the output of two backendsGGML_API bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t backend2, struct ggml_cgraph * graph, ggml_backend_eval_callback callback, void * user_data, struct ggml_tensor const * const * test_nodes, size_t num_test_nodes);- // returns true for ops that may require additional memory for fleeting data on some backends,- // i.e. the backend's get_alloc_size may return more than ggml_nbytes for the output tensor- GGML_API bool ggml_backend_op_alloc_size_may_expand(enum ggml_op op);-// Tensor initializationGGML_API enum ggml_status ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, void * addr);GGML_API enum ggml_status ggml_backend_view_init(struct ggml_tensor * tensor);diff --git a/ggml/src/ggml-backend-impl.h b/ggml/src/ggml-backend-impl.hindex 56f0090cce..ef05905cf9 100644--- a/ggml/src/ggml-backend-impl.h+++ b/ggml/src/ggml-backend-impl.h@@ -34,6 +34,11 @@ extern "C" {void * context;};+ // [TAG_ALLOC_SIZE_EXPAND]+ // returns true for ops that may require additional memory for fleeting data on some backends,+ // i.e. the backend buffer type's get_alloc_size may return more than ggml_nbytes for the output tensor+ GGML_API bool ggml_op_alloc_size_may_expand(enum ggml_op op);+//// Backend buffer//diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cppindex ffe20b9d05..6862128e63 100644--- a/ggml/src/ggml-backend.cpp+++ b/ggml/src/ggml-backend.cpp@@ -71,7 +71,7 @@ size_t ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type_t buft, const sGGML_ASSERT(size <= ggml_nbytes(tensor) ||ggml_op_is_empty(tensor->op) ||ggml_is_quantized(tensor->type) || // [TAG_ALLOC_SIZE_EXPAND]- ggml_backend_op_alloc_size_may_expand(tensor->op));+ ggml_op_alloc_size_may_expand(tensor->op));return size;}@@ -2109,10 +2109,7 @@ ggml_backend_t ggml_backend_sched_get_tensor_backend(ggml_backend_sched_t sched,// utils-// [TAG_ALLOC_SIZE_EXPAND]-// returns true for ops that may require additional memory for fleeting data on some backends,-// i.e. the backend's get_alloc_size may return more than ggml_nbytes for the output tensor-bool ggml_backend_op_alloc_size_may_expand(enum ggml_op op) {+bool ggml_op_alloc_size_may_expand(enum ggml_op op) {switch (op) {case GGML_OP_FLASH_ATTN_EXT:case GGML_OP_MUL_MAT:diff --git a/ggml/src/ggml-rpc/ggml-rpc.cpp b/ggml/src/ggml-rpc/ggml-rpc.cppindex a97db24e62..cc7d720693 100644--- a/ggml/src/ggml-rpc/ggml-rpc.cpp+++ b/ggml/src/ggml-rpc/ggml-rpc.cpp@@ -835,7 +835,7 @@ static size_t ggml_backend_rpc_buffer_type_get_alloc_size(ggml_backend_buffer_ty// [TAG_ALLOC_SIZE_EXPAND]// ops that may require additional memory for fleeting data on certain backends// ref: https://github.com/ggml-org/llama.cpp/pull/15966- rpc_get |= ggml_backend_op_alloc_size_may_expand(tensor->op);+ rpc_get |= ggml_op_alloc_size_may_expand(tensor->op);if (rpc_get) {ggml_backend_rpc_buffer_type_context * buft_ctx = (ggml_backend_rpc_buffer_type_context *)buft->context;diff --git a/scripts/sync-ggml.last b/scripts/sync-ggml.lastindex 601c1108bb..7b44a311ab 100644--- a/scripts/sync-ggml.last+++ b/scripts/sync-ggml.last@@ -1 +1 @@-36da57138425487184aa1da2eee2cde155909c6f+e91ded11bdcd78c42f9c8d3978ff6686eb4c1226