Skip to content

LLM inference in C/C++

C++ ggml

ggml: fix backend split scheduler race condition (#26040)

Ruben Ortlam849798132173
diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp
index d5ba5b5cea..e519bdf50a 100644
--- a/ggml/src/ggml-backend.cpp
+++ b/ggml/src/ggml-backend.cpp
@@ -1601,11 +1601,23 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s
std::vector<int32_t> ids;
std::vector<ggml_bitset_t> used_ids;
+ int prev_backend_id = -1;
+
for (int split_id = 0; split_id < sched->n_splits; split_id++) {
struct ggml_backend_sched_split * split = &splits[split_id];
int split_backend_id = split->backend_id;
ggml_backend_t split_backend = sched->backends[split_backend_id];
+ // ensure the previous split's async work has completed before we start
+ // this split, the allocator may have reused buffer regions across splits
+ if (split->n_inputs == 0 && prev_backend_id >= 0 && prev_backend_id != split_backend_id) {
+ if (sched->events[prev_backend_id][sched->cur_copy] != NULL) {
+ ggml_backend_event_synchronize(sched->events[prev_backend_id][sched->cur_copy]);
+ } else {
+ ggml_backend_synchronize(sched->backends[prev_backend_id]);
+ }
+ }
+
// copy the input tensors to the split backend
for (int input_id = 0; input_id < split->n_inputs; input_id++) {
ggml_backend_t input_backend = ggml_backend_sched_get_tensor_backend(sched, split->inputs[input_id]);
@@ -1768,12 +1780,12 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s
}
}
- // record the event of this copy
- if (split->n_inputs > 0) {
- if (sched->events[split_backend_id][sched->cur_copy] != NULL) {
- ggml_backend_event_record(sched->events[split_backend_id][sched->cur_copy], split_backend);
- }
+ // record the event of this split
+ if (sched->events[split_backend_id][sched->cur_copy] != NULL) {
+ ggml_backend_event_record(sched->events[split_backend_id][sched->cur_copy], split_backend);
}
+
+ prev_backend_id = split_backend_id;
}
return GGML_STATUS_SUCCESS;