LLM inference in C/C++
C++ ggml
tensor-split meta backend fixes (#26502)
Max Krasnyanskyd59d455fd8ea
diff --git a/ggml/src/ggml-backend-impl.h b/ggml/src/ggml-backend-impl.hindex 9c56ec30c5..40cea024c3 100644--- a/ggml/src/ggml-backend-impl.h+++ b/ggml/src/ggml-backend-impl.h@@ -83,6 +83,7 @@ extern "C" {GGML_API ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers);GGML_API bool ggml_backend_buffer_is_multi_buffer(ggml_backend_buffer_t buffer);GGML_API void ggml_backend_multi_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage);+ GGML_API void ggml_backend_meta_buffer_set_usage (ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage);//// Backend (meta)diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cppindex 7654ea1f30..775ae99267 100644--- a/ggml/src/ggml-backend-meta.cpp+++ b/ggml/src/ggml-backend-meta.cpp@@ -1118,7 +1118,6 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(}static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(const struct ggml_tensor * tensor, bool assume_sync) {- GGML_ASSERT(ggml_backend_buffer_is_meta(tensor->buffer));ggml_backend_meta_buffer_context * buf_ctx = (ggml_backend_meta_buffer_context *) tensor->buffer->context;return ggml_backend_meta_get_split_state(buf_ctx->get_simple_tensor_container(tensor), tensor, assume_sync);}@@ -1178,7 +1177,15 @@ static enum ggml_status ggml_backend_meta_buffer_init_tensor_impl(ggml_backend_mt_ij->flags = tensor->flags;memcpy(t_ij->op_params, tensor->op_params, sizeof(tensor->op_params));ggml_set_name(t_ij, tensor->name);+t_ij->buffer = simple_buf;+ if (simple_buf) {+ // the backend that owns the buffer will set .extra+ ggml_backend_buffer_init_tensor(simple_buf, t_ij);+ } else {+ t_ij->extra = tensor->extra;+ }+t_ij->view_src = tensor->view_src;t_ij->view_offs = tensor->view_offs;if (t_ij->view_src != nullptr && ggml_backend_buffer_is_meta(t_ij->view_src->buffer)) {@@ -1209,7 +1216,6 @@ static enum ggml_status ggml_backend_meta_buffer_init_tensor_impl(ggml_backend_mt_ij->data = (char *) ggml_backend_buffer_get_base(simple_buf)+ size_t(tensor->data) - size_t(ggml_backend_buffer_get_base(tensor->buffer));}- t_ij->extra = tensor->extra;for (int i = 0; i < GGML_MAX_SRC; i++) {t_ij->src[i] = tensor->src[i];if (tensor->src[i] == tensor) {@@ -1502,6 +1508,16 @@ bool ggml_backend_buffer_is_meta(ggml_backend_buffer_t buf) {return buf != nullptr && buf->iface.free_buffer == ggml_backend_meta_buffer_iface.free_buffer;}+void ggml_backend_meta_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage) {+ GGML_ASSERT(ggml_backend_buffer_is_meta(buffer));+ ggml_backend_meta_buffer_context * buf_ctx = (ggml_backend_meta_buffer_context *) buffer->context;+ for (size_t i = 0; i < buf_ctx->bufs.size(); i++) {+ if (buf_ctx->bufs[i]) {+ ggml_backend_buffer_set_usage(buf_ctx->bufs[i].get(), usage);+ }+ }+}+static ggml_backend_buffer_t ggml_backend_meta_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {const size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft);diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cppindex f6fb91798c..d5ba5b5cea 100644--- a/ggml/src/ggml-backend.cpp+++ b/ggml/src/ggml-backend.cpp@@ -182,6 +182,8 @@ void ggml_backend_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backe// FIXME: add a generic callback to the buffer interfaceif (ggml_backend_buffer_is_multi_buffer(buffer)) {ggml_backend_multi_buffer_set_usage(buffer, usage);+ } else if (ggml_backend_buffer_is_meta(buffer)) {+ ggml_backend_meta_buffer_set_usage(buffer, usage);}}