Skip to content

LLM inference in C/C++

C++ ggml

CUDA: Allow concurrent streams per split for multi-GPU (#28198)

Tanner Bruhn0ba6499c3ba7
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
index f4af82688a..45e9537f0e 100644
--- a/ggml/src/ggml-cuda/ggml-cuda.cu
+++ b/ggml/src/ggml-cuda/ggml-cuda.cu
@@ -4542,10 +4542,12 @@ static void ggml_backend_cuda_graph_optimize(ggml_backend_t backend, ggml_cgraph
ggml_cuda_stream_context & stream_context = cuda_ctx->stream_context();
stream_context.reset();
- if (!use_cuda_graph || ggml_backend_cuda_get_device_count() != 1) {
+ if (!use_cuda_graph) {
return;
}
+ ggml_cuda_set_device(cuda_ctx->device);
+
// number of out-degrees for a particular node
std::unordered_map<const ggml_tensor *, int> fan_out;
// reverse mapping of node to index in the cgraph