LLM inference in C/C++
C++ ggml
Revert "sched : reintroduce less synchronizations during split compute (#20793)" (#25138)
Oliver Simons86b94708f224
diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cppindex 644130cd13..87615921c0 100644--- a/ggml/src/ggml-backend.cpp+++ b/ggml/src/ggml-backend.cpp@@ -1551,8 +1551,6 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t sint split_backend_id = split->backend_id;ggml_backend_t split_backend = sched->backends[split_backend_id];- ggml_backend_synchronize(split_backend);-// copy the input tensors to the split backendfor (int input_id = 0; input_id < split->n_inputs; input_id++) {ggml_backend_t input_backend = ggml_backend_sched_get_tensor_backend(sched, split->inputs[input_id]);@@ -1563,15 +1561,15 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s// inputs from the user must be copied immediately to prevent the user overwriting the data before the copy is doneif (sched->events[split_backend_id][sched->cur_copy] != NULL) {ggml_backend_event_synchronize(sched->events[split_backend_id][sched->cur_copy]);- } else if (!split_backend->iface.cpy_tensor_async) {+ } else {ggml_backend_synchronize(split_backend);}- ggml_backend_tensor_copy_async(input_backend, split_backend, input, input_cpy);+ ggml_backend_tensor_copy(input, input_cpy);} else {// wait for the split backend to finish using the input before overwriting itif (sched->events[split_backend_id][sched->cur_copy] != NULL) {ggml_backend_event_wait(split_backend, sched->events[split_backend_id][sched->cur_copy]);- } else if (!split_backend->iface.cpy_tensor_async) {+ } else {ggml_backend_synchronize(split_backend);}@@ -1676,8 +1674,6 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s}}- ggml_backend_synchronize(split_backend);-if (!sched->callback_eval) {enum ggml_status ec = ggml_backend_graph_compute_async(split_backend, &split->graph);if (ec != GGML_STATUS_SUCCESS) {diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuindex f3fb32452d..cca70592f8 100644--- a/ggml/src/ggml-cuda/ggml-cuda.cu+++ b/ggml/src/ggml-cuda/ggml-cuda.cu@@ -3192,24 +3192,11 @@ static bool ggml_backend_cuda_cpy_tensor_async(ggml_backend_t backend_src, ggml_ggml_backend_buffer_t buf_src = src->view_src ? src->view_src->buffer : src->buffer;ggml_backend_buffer_t buf_dst = dst->view_src ? dst->view_src->buffer : dst->buffer;- // Enables async copies from CPU to CUDA, instead of only CUDA-to-CUDA- // Excluding this path for HIP and MUSA as a precaution.- // According to the summary in https://github.com/ggml-org/llama.cpp/pull/20793#issuecomment-4275794315, this change is not beneficial for hip anyways.- // Additionally, there is a lot of anectodal evidence that hip/musa stream behavior might not always 1:1 match CUDA behavior.- // e.g. https://github.com/ROCm/rocm-systems/issues/5109- // It thus makes sense to exclude this path for HIP and MUSA. This PR was not aimed these backends, the majority of testing happened on CUDA.- // This can be revisited in the future if enabling copy_from_host benefits hip/MUSA, and if the PR author can extensively test on these backends.-#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)- const bool copy_from_host = false;-#else- const bool copy_from_host = ggml_backend_buffer_is_host(buf_src) && ggml_backend_dev_type(backend_src->device) == GGML_BACKEND_DEVICE_TYPE_CPU;-#endif-- if (!(copy_from_host || ggml_backend_is_cuda(backend_src)) || !ggml_backend_is_cuda(backend_dst)) {+ if (!ggml_backend_is_cuda(backend_src) || !ggml_backend_is_cuda(backend_dst)) {return false;}- if (!(copy_from_host || ggml_backend_buffer_is_cuda(buf_src)) || !ggml_backend_buffer_is_cuda(buf_dst)) {+ if (!ggml_backend_buffer_is_cuda(buf_src) || !ggml_backend_buffer_is_cuda(buf_dst)) {return false;}@@ -3220,17 +3207,14 @@ static bool ggml_backend_cuda_cpy_tensor_async(ggml_backend_t backend_src, ggml_ggml_backend_cuda_buffer_context * buf_ctx_src = (ggml_backend_cuda_buffer_context *) buf_src->context;ggml_backend_cuda_buffer_context * buf_ctx_dst = (ggml_backend_cuda_buffer_context *) buf_dst->context;- if ((copy_from_host && cuda_ctx_dst->device != buf_ctx_dst->device) ||- !copy_from_host && (cuda_ctx_src->device != buf_ctx_src->device || cuda_ctx_dst->device != buf_ctx_dst->device)) {+ if (cuda_ctx_src->device != buf_ctx_src->device || cuda_ctx_dst->device != buf_ctx_dst->device) {#ifndef NDEBUGGGML_LOG_DEBUG("%s: backend and buffer devices do not match\n", __func__);#endif // NDEBUGreturn false;}- if (copy_from_host) {- CUDA_CHECK(cudaMemcpyAsync(dst->data, src->data, ggml_nbytes(dst), cudaMemcpyHostToDevice, cuda_ctx_dst->stream()));- } else if (backend_src != backend_dst) {+ if (backend_src != backend_dst) {// copy on src streamif (cuda_ctx_src->device == cuda_ctx_dst->device) {CUDA_CHECK(cudaMemcpyAsync(dst->data, src->data, ggml_nbytes(dst), cudaMemcpyDeviceToDevice, cuda_ctx_src->stream()));