LLM inference in C/C++
C++ ggml
ggml: allow passing alloc dependencies in graph_optimize (#27301)
Aman Gupta57291f2644af
diff --git a/ggml/src/ggml-backend-impl.h b/ggml/src/ggml-backend-impl.hindex 40cea024c3..56f0090cce 100644--- a/ggml/src/ggml-backend-impl.h+++ b/ggml/src/ggml-backend-impl.h@@ -103,6 +103,16 @@ extern "C" {// Backend (stream)//+ // passed to graph_optimize so the backend can add allocation dependencies:+ // if the backend executes parts of the graph out of order (e.g. on concurrent streams),+ // it must keep the affected tensors allocated until a node where execution is known to have joined+ struct ggml_backend_graph_optimize_params {+ // keep `tensor` allocated at least until `until` (a node of the same graph) has been computed+ // can be called multiple times for the same tensor: the longest lifetime applies+ void (*add_alloc_dep)(void * user_data, struct ggml_tensor * tensor, struct ggml_tensor * until);+ void * user_data;+ };+struct ggml_backend_i {const char * (*get_name)(ggml_backend_t backend);@@ -137,7 +147,7 @@ extern "C" {void (*event_wait) (ggml_backend_t backend, ggml_backend_event_t event);// (optional) sort/optimize the nodes in the graph- void (*graph_optimize) (ggml_backend_t backend, struct ggml_cgraph * cgraph);+ void (*graph_optimize) (ggml_backend_t backend, struct ggml_cgraph * cgraph, struct ggml_backend_graph_optimize_params * params);};struct ggml_backend {diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cppindex e519bdf50a..78eb10dfe9 100644--- a/ggml/src/ggml-backend.cpp+++ b/ggml/src/ggml-backend.cpp@@ -20,6 +20,7 @@#include <stdlib.h>#include <string.h>#include <algorithm>+#include <unordered_map>#include <vector>#ifdef __APPLE__@@ -558,10 +559,10 @@ void ggml_backend_event_wait(ggml_backend_t backend, ggml_backend_event_t event)backend->iface.event_wait(backend, event);}-static void ggml_backend_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * cgraph) {+static void ggml_backend_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * cgraph, struct ggml_backend_graph_optimize_params * params) {GGML_ASSERT(backend);if (backend->iface.graph_optimize != NULL) {- backend->iface.graph_optimize(backend, cgraph);+ backend->iface.graph_optimize(backend, cgraph, params);}}@@ -1441,11 +1442,40 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgrasched->prev_leaf_backend_ids = tmp;}+ // optimize the split graphs and collect the allocation dependencies added by the backends+ // this needs to happen before we make graph_copy, so they are in sync+ // TODO: this may create many small allocations in the scheduler, restructure to use a flat array+ std::unordered_map<ggml_tensor *, std::vector<ggml_tensor *>> alloc_deps;++ struct ggml_backend_graph_optimize_params opt_params = {+ /* .add_alloc_dep = */ [](void * user_data, ggml_tensor * tensor, ggml_tensor * until) {+ auto & deps = *(std::unordered_map<ggml_tensor *, std::vector<ggml_tensor *>> *) user_data;+ std::vector<ggml_tensor *> & keep = deps[until];+ if (std::find(keep.begin(), keep.end(), tensor) == keep.end()) {+ keep.push_back(tensor);+ }+ },+ /* .user_data = */ &alloc_deps,+ };++ for (int i = 0; i < sched->n_splits; i++) {+ struct ggml_backend_sched_split * split = &sched->splits[i];+ split->graph = ggml_graph_view(graph, split->i_start, split->i_end);++ ggml_backend_graph_optimize(sched->backends[split->backend_id], &split->graph, &opt_params);+ }++ // each dep is added to graph_copy as a GGML_OP_NONE node with the kept tensors as srcs+ int n_dep_nodes = 0;+ for (const auto & it : alloc_deps) {+ n_dep_nodes += (it.second.size() + GGML_MAX_SRC - 1) / GGML_MAX_SRC;+ }+int total_inputs = sched->n_graph_inputs;for (int i = 0; i < sched->n_splits; i++) {total_inputs += sched->splits[i].n_inputs;}- int graph_size = std::max(graph->n_nodes, graph->n_leafs) + total_inputs * 2 * sched->n_copies;+ int graph_size = std::max(graph->n_nodes, graph->n_leafs) + total_inputs * 2 * sched->n_copies + n_dep_nodes;// remember the actual graph_size for performing reallocation checks later [GGML_SCHED_DEBUG_REALLOC]sched->debug_prev_graph_size = sched->debug_graph_size;@@ -1463,13 +1493,10 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgrastruct ggml_cgraph * graph_copy = &sched->graph;+ int n_dep_nodes_added = 0;+for (int i = 0; i < sched->n_splits; i++) {struct ggml_backend_sched_split * split = &sched->splits[i];- split->graph = ggml_graph_view(graph, split->i_start, split->i_end);-- // Optimize this split of the graph. This needs to happen before we make graph_copy,- // so they are in sync.- ggml_backend_graph_optimize(sched->backends[split->backend_id], &split->graph);// add inputs to the graph copy so that they are allocated by ggml-alloc at the start of the splitfor (int j = 0; j < split->n_inputs; j++) {@@ -1494,9 +1521,32 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraassert(graph_copy->size > graph_copy->n_nodes);sched->node_backend_ids[graph_copy->n_nodes] = tensor_backend_id(graph->nodes[j]);graph_copy->nodes[graph_copy->n_nodes++] = graph->nodes[j];++ if (alloc_deps.empty()) {+ continue;+ }++ // add a dependency node so that the kept tensors are not freed before this node is computed+ auto it = alloc_deps.find(graph->nodes[j]);+ if (it != alloc_deps.end()) {+ const std::vector<ggml_tensor *> & keep = it->second;+ for (size_t k = 0; k < keep.size(); k += GGML_MAX_SRC) {+ struct ggml_tensor * dep = ggml_view_tensor(sched->ctx, keep[k]);+ for (size_t s = 0; s < GGML_MAX_SRC && k + s < keep.size(); s++) {+ dep->src[s] = keep[k + s];+ }+ assert(graph_copy->size > graph_copy->n_nodes);+ sched->node_backend_ids[graph_copy->n_nodes] = split->backend_id;+ graph_copy->nodes[graph_copy->n_nodes++] = dep;+ n_dep_nodes_added++;+ }+ }}}+ // a mismatch means a backend added a dep with an `until` tensor that is not a node of the optimized graph+ GGML_ASSERT(n_dep_nodes_added == n_dep_nodes);+if (sched->n_copies > 1) {// add input copies as leafs so that they are allocated firstfor (int i = 0; i < sched->n_graph_inputs; i++) {diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuindex 2456f7dcc6..bd9754c2ff 100644--- a/ggml/src/ggml-cuda/ggml-cuda.cu+++ b/ggml/src/ggml-cuda/ggml-cuda.cu@@ -4328,7 +4328,9 @@ static void ggml_backend_cuda_event_wait(ggml_backend_t backend, ggml_backend_ev}}-static void ggml_backend_cuda_graph_optimize(ggml_backend_t backend, ggml_cgraph * cgraph) {+static void ggml_backend_cuda_graph_optimize(ggml_backend_t backend, ggml_cgraph * cgraph, ggml_backend_graph_optimize_params * params) {+ GGML_UNUSED(params);+ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;#ifdef USE_CUDA_GRAPHdiff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cppindex 53e8607559..e7dcdc3d55 100644--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp@@ -4984,7 +4984,9 @@ static std::vector<int> ggml_hexagon_graph_optimize_reorder(const std::vector<htreturn res;}-static void ggml_backend_hexagon_graph_optimize(ggml_backend_t backend, ggml_cgraph * gf) {+static void ggml_backend_hexagon_graph_optimize(ggml_backend_t backend, ggml_cgraph * gf, ggml_backend_graph_optimize_params * params) {+ GGML_UNUSED(params);+const int n = gf->n_nodes;constexpr int MAX_FUSE = 16;diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal.cppindex 9756d47050..4d58dc821c 100644--- a/ggml/src/ggml-metal/ggml-metal.cpp+++ b/ggml/src/ggml-metal/ggml-metal.cpp@@ -558,7 +558,9 @@ static void ggml_backend_metal_event_wait(ggml_backend_t backend, ggml_backend_eggml_metal_event_wait(ctx, ev);}-static void ggml_backend_metal_graph_optimize(ggml_backend_t backend, ggml_cgraph * cgraph) {+static void ggml_backend_metal_graph_optimize(ggml_backend_t backend, ggml_cgraph * cgraph, ggml_backend_graph_optimize_params * params) {+ GGML_UNUSED(params);+ggml_metal_t ctx = (ggml_metal_t)backend->context;ggml_metal_graph_optimize(ctx, cgraph);diff --git a/ggml/src/ggml-virtgpu/ggml-backend.cpp b/ggml/src/ggml-virtgpu/ggml-backend.cppindex 12756c9282..996c57e358 100644--- a/ggml/src/ggml-virtgpu/ggml-backend.cpp+++ b/ggml/src/ggml-virtgpu/ggml-backend.cpp@@ -17,7 +17,8 @@ static ggml_status ggml_backend_remoting_graph_compute(ggml_backend_t backend, greturn apir_backend_graph_compute(gpu, cgraph);}-static void ggml_backend_remoting_graph_optimize(ggml_backend_t backend, ggml_cgraph * cgraph) {+static void ggml_backend_remoting_graph_optimize(ggml_backend_t backend, ggml_cgraph * cgraph, ggml_backend_graph_optimize_params * params) {+ UNUSED(params);virtgpu * gpu = DEV_TO_GPU(backend->device);#if trueUNUSED(gpu);diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cppindex 39b4cd3598..8fbb1359f4 100644--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp@@ -17795,8 +17795,9 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg}// Sort the graph for improved parallelism.-static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * graph)+static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * graph, struct ggml_backend_graph_optimize_params * params){+ GGML_UNUSED(params);VK_LOG_DEBUG("ggml_vk_graph_optimize(" << graph->n_nodes << " nodes)");ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;diff --git a/tests/test-alloc.cpp b/tests/test-alloc.cppindex 6d5428493e..8f1a98aa03 100644--- a/tests/test-alloc.cpp+++ b/tests/test-alloc.cpp@@ -19,6 +19,8 @@ struct dummy_backend_context {size_t alignment = 8;ggml_backend_buffer_i buffer_interface;+ ggml_backend_device device;+ ggml_backend backend;std::vector<ggml_backend_buffer_t> buffers;size_t allocated_total() const {@@ -83,7 +85,27 @@ static void dummy_backend_buffer_get_tensor(ggml_backend_buffer_t, const ggml_testatic void dummy_backend_buffer_clear(ggml_backend_buffer_t, uint8_t) {}-// dummy_backend (not really a full backend, just provides what gallocr needs)+// ggml_backend_device interface++static enum ggml_backend_dev_type dummy_backend_device_get_type(ggml_backend_dev_t) {+ return GGML_BACKEND_DEVICE_TYPE_CPU;+}++static bool dummy_backend_device_supports_op(ggml_backend_dev_t, const ggml_tensor *) {+ return true;+}++static bool dummy_backend_device_supports_buft(ggml_backend_dev_t device, ggml_backend_buffer_type_t buft) {+ return device->context == buft->context;+}++// ggml_backend interface++static const char * dummy_backend_get_name(ggml_backend_t) {+ return "dummy_backend";+}++// dummy_backendstruct dummy_backend {std::unique_ptr<dummy_backend_context> context;@@ -104,6 +126,16 @@ static dummy_backend dummy_backend_init(size_t max_buffer_size, size_t alignmentb.context->buffer_interface.get_tensor = dummy_backend_buffer_get_tensor;b.context->buffer_interface.clear = dummy_backend_buffer_clear;+ b.context->device.context = b.context.get();+ b.context->device.iface.get_type = dummy_backend_device_get_type;+ b.context->device.iface.supports_op = dummy_backend_device_supports_op;+ b.context->device.iface.supports_buft = dummy_backend_device_supports_buft;++ b.context->backend.context = b.context.get();+ b.context->backend.device = &b.context->device;+ b.context->backend.iface.get_name = dummy_backend_get_name;++ b.buffer_type.device = &b.context->device;b.buffer_type.context = b.context.get();b.buffer_type.iface.get_name = dummy_backend_buffer_type_get_name;b.buffer_type.iface.alloc_buffer = dummy_backend_buffer_type_alloc_buffer;@@ -583,6 +615,41 @@ static void test_reallocation() {}}+static void test_backend_graph_optimize(ggml_backend_t, ggml_cgraph * graph, ggml_backend_graph_optimize_params * params) {+ GGML_ASSERT(graph->n_nodes == 3);+ params->add_alloc_dep(params->user_data, graph->nodes[0], graph->nodes[2]);+}++static bool graph_reuses_allocation(bool add_alloc_dep) {+ auto [ctx, graph, ctx_ptr] = make_context();++ ggml_tensor * x[4];+ x[0] = make_input_with_size(ctx, 16);+ x[1] = ggml_scale(ctx, x[0], 2.0f);+ x[2] = ggml_scale(ctx, x[1], 2.0f);+ x[3] = ggml_scale(ctx, x[2], 2.0f);++ ggml_set_output(x[3]);+ ggml_build_forward_expand(graph, x[3]);++ dummy_backend backend = dummy_backend_init(SIZE_MAX);+ if (add_alloc_dep) {+ backend.context->backend.iface.graph_optimize = test_backend_graph_optimize;+ }++ ggml_backend_t backend_ptr = &backend.context->backend;+ ggml_backend_buffer_type_t buft = &backend.buffer_type;+ ggml_backend_sched_ptr sched(ggml_backend_sched_new(&backend_ptr, &buft, 1, 8, false, true));+ GGML_ASSERT(ggml_backend_sched_alloc_graph(sched.get(), graph));++ return x[1]->data == x[2]->data;+}++static void test_graph_optimize_alloc_dep() {+ GGML_ASSERT(graph_reuses_allocation(false));+ GGML_ASSERT(!graph_reuses_allocation(true));+}+static void run(const char * name, void (*f)()) {printf("%s ", name);fflush(stdout);@@ -604,5 +671,6 @@ int main() {run("test_multiple_buffer_types", test_multiple_buffer_types);run("test_buffer_size_zero", test_buffer_size_zero);run("test_reallocation", test_reallocation);+ run("test_graph_optimize_alloc_dep", test_graph_optimize_alloc_dep);return 0;}