Skip to content

LLM inference in C/C++

C++ ggml

ggml : collect all input tensors into graph_inputs (#29634)

Georgi Gerganov8019dc563b1e
diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp
index 20bf965017..273dc92b25 100644
--- a/ggml/src/ggml-backend.cpp
+++ b/ggml/src/ggml-backend.cpp
@@ -1372,30 +1372,6 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra
const int src_backend_id = sched->hv_tensor_backend_ids[src_id];
GGML_ASSERT(src_backend_id != -1); // all inputs should be assigned by now
- if (src->flags & GGML_TENSOR_FLAG_INPUT && sched->n_copies > 1) {
- if (tensor_id_copy(src_id, src_backend_id, 0) == NULL) {
- ggml_backend_t backend = sched->backends[src_backend_id];
- for (int c = 0; c < sched->n_copies; c++) {
- struct ggml_tensor * tensor_copy;
- if (c == sched->cur_copy) {
- tensor_copy = src; // use the original tensor as the current copy
- } else {
- tensor_copy = ggml_dup_tensor_layout(sched->ctx, src);
- ggml_format_name(tensor_copy, "%s#%s#%d", ggml_backend_name(backend), src->name, c);
- }
- ggml_set_input(tensor_copy);
- ggml_set_output(tensor_copy); // prevent ggml-alloc from overwriting the tensor
- tensor_id_copy(src_id, src_backend_id, c) = tensor_copy;
- SET_CAUSE(tensor_copy, "4.cpy");
- }
- int n_graph_inputs = sched->n_graph_inputs++;
- if (n_graph_inputs >= sched->graph_inputs_capacity) {
- ggml_backend_sched_graph_inputs_grow(sched);
- }
- sched->graph_inputs[n_graph_inputs] = src;
- }
- }
-
if (src_backend_id != cur_backend_id && !ggml_backend_sched_buffer_supported(sched, src, cur_backend_id)) {
// create a copy of the input in the split's backend
if (tensor_id_copy(src_id, cur_backend_id, 0) == NULL) {
@@ -1428,6 +1404,46 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra
ggml_backend_sched_print_assignments(sched, graph);
}
+ // pass 6: collect all input tensors into graph_inputs
+ // this includes inputs not consumed by any node (e.g. the embeddings input of a text-only batch) so that
+ // the graph composition does not depend on which inputs are used, which would otherwise cause graph
+ // reallocations when switching between different types of batches [GGML_SCHED_DEBUG_REALLOC]
+ if (sched->n_copies > 1) {
+ for (int i = 0; i < graph->n_leafs; i++) {
+ struct ggml_tensor * leaf = graph->leafs[i];
+ if ((leaf->flags & GGML_TENSOR_FLAG_INPUT) == 0) {
+ continue;
+ }
+
+ const size_t leaf_id = hash_id(leaf);
+ const int leaf_backend_id = tensor_backend_id(leaf);
+ GGML_ASSERT(leaf_backend_id != -1); // all leafs should be assigned by now
+
+ if (tensor_id_copy(leaf_id, leaf_backend_id, 0) == NULL) {
+ ggml_backend_t backend = sched->backends[leaf_backend_id];
+ for (int c = 0; c < sched->n_copies; c++) {
+ struct ggml_tensor * tensor_copy;
+ if (c == sched->cur_copy) {
+ tensor_copy = leaf; // use the original tensor as the current copy
+ } else {
+ tensor_copy = ggml_dup_tensor_layout(sched->ctx, leaf);
+ ggml_format_name(tensor_copy, "%s#%s#%d", ggml_backend_name(backend), leaf->name, c);
+ }
+ ggml_set_input(tensor_copy);
+ ggml_set_output(tensor_copy); // prevent ggml-alloc from overwriting the tensor
+ tensor_id_copy(leaf_id, leaf_backend_id, c) = tensor_copy;
+ SET_CAUSE(tensor_copy, "6.cpy");
+ }
+ }
+
+ int n_graph_inputs = sched->n_graph_inputs++;
+ if (n_graph_inputs >= sched->graph_inputs_capacity) {
+ ggml_backend_sched_graph_inputs_grow(sched);
+ }
+ sched->graph_inputs[n_graph_inputs] = leaf;
+ }
+ }
+
// swap node_backend_ids and leaf _backend_ids with prevs
{
int * tmp = sched->node_backend_ids;