LLM inference in C/C++
C++ ggml
CUDA: Allow concurrent streams per split for multi-GPU (#28198)
Tanner Bruhn0ba6499c3ba7
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuindex f4af82688a..45e9537f0e 100644--- a/ggml/src/ggml-cuda/ggml-cuda.cu+++ b/ggml/src/ggml-cuda/ggml-cuda.cu@@ -4542,10 +4542,12 @@ static void ggml_backend_cuda_graph_optimize(ggml_backend_t backend, ggml_cgraphggml_cuda_stream_context & stream_context = cuda_ctx->stream_context();stream_context.reset();- if (!use_cuda_graph || ggml_backend_cuda_get_device_count() != 1) {+ if (!use_cuda_graph) {return;}+ ggml_cuda_set_device(cuda_ctx->device);+// number of out-degrees for a particular nodestd::unordered_map<const ggml_tensor *, int> fan_out;// reverse mapping of node to index in the cgraph