LLM inference in C/C++
C++ ggml
ggml: fix backend split scheduler race condition (#26040)
Ruben Ortlam849798132173
diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cppindex d5ba5b5cea..e519bdf50a 100644--- a/ggml/src/ggml-backend.cpp+++ b/ggml/src/ggml-backend.cpp@@ -1601,11 +1601,23 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t sstd::vector<int32_t> ids;std::vector<ggml_bitset_t> used_ids;+ int prev_backend_id = -1;+for (int split_id = 0; split_id < sched->n_splits; split_id++) {struct ggml_backend_sched_split * split = &splits[split_id];int split_backend_id = split->backend_id;ggml_backend_t split_backend = sched->backends[split_backend_id];+ // ensure the previous split's async work has completed before we start+ // this split, the allocator may have reused buffer regions across splits+ if (split->n_inputs == 0 && prev_backend_id >= 0 && prev_backend_id != split_backend_id) {+ if (sched->events[prev_backend_id][sched->cur_copy] != NULL) {+ ggml_backend_event_synchronize(sched->events[prev_backend_id][sched->cur_copy]);+ } else {+ ggml_backend_synchronize(sched->backends[prev_backend_id]);+ }+ }+// copy the input tensors to the split backendfor (int input_id = 0; input_id < split->n_inputs; input_id++) {ggml_backend_t input_backend = ggml_backend_sched_get_tensor_backend(sched, split->inputs[input_id]);@@ -1768,12 +1780,12 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s}}- // record the event of this copy- if (split->n_inputs > 0) {- if (sched->events[split_backend_id][sched->cur_copy] != NULL) {- ggml_backend_event_record(sched->events[split_backend_id][sched->cur_copy], split_backend);- }+ // record the event of this split+ if (sched->events[split_backend_id][sched->cur_copy] != NULL) {+ ggml_backend_event_record(sched->events[split_backend_id][sched->cur_copy], split_backend);}++ prev_backend_id = split_backend_id;}return GGML_STATUS_SUCCESS;