LLM inference in C/C++
C++ ggml
CUDA: Clear CUDA graphs after legacy memory pool flush (#26771)
Bálint Bokros13786fbb3a1a
diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuhindex fea1cf9c40..049100fe38 100644--- a/ggml/src/ggml-cuda/common.cuh+++ b/ggml/src/ggml-cuda/common.cuh@@ -1551,11 +1551,11 @@ struct ggml_backend_cuda_context {// poolstd::unique_ptr<ggml_cuda_pool> pools[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS];- static std::unique_ptr<ggml_cuda_pool> new_pool_for_device(int device, int stream_no);+ static std::unique_ptr<ggml_cuda_pool> new_pool_for_device(ggml_backend_cuda_context & ctx, int device, int stream_no);ggml_cuda_pool & pool(int device) {if (pools[device][curr_stream_no] == nullptr) {- pools[device][curr_stream_no] = new_pool_for_device(device, curr_stream_no);+ pools[device][curr_stream_no] = new_pool_for_device(*this, device, curr_stream_no);}return *pools[device][curr_stream_no];}diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuindex 2eb5e203ef..d9b18cc08d 100644--- a/ggml/src/ggml-cuda/ggml-cuda.cu+++ b/ggml/src/ggml-cuda/ggml-cuda.cu@@ -457,6 +457,7 @@ const ggml_cuda_device_info & ggml_cuda_info() {struct ggml_cuda_pool_leg : public ggml_cuda_pool {static const int MAX_BUFFERS = 256;+ ggml_backend_cuda_context & ctx;int device;struct ggml_cuda_buffer {void * ptr = nullptr;@@ -466,7 +467,8 @@ struct ggml_cuda_pool_leg : public ggml_cuda_pool {ggml_cuda_buffer buffer_pool[MAX_BUFFERS] = {};size_t pool_size = 0;- explicit ggml_cuda_pool_leg(int device) :+ ggml_cuda_pool_leg(ggml_backend_cuda_context & ctx, int device) :+ ctx(ctx),device(device) {}@@ -488,6 +490,20 @@ struct ggml_cuda_pool_leg : public ggml_cuda_pool {}}+ // captured CUDA graphs can still use freed buffers, so they must be captured again+ void invalidate_cuda_graphs() {+#ifdef USE_CUDA_GRAPH+ int n_captured = 0;+ for (const auto & [key, graph] : ctx.cuda_graphs) {+ n_captured += graph->instance != nullptr;+ }+ if (n_captured > 0) {+ GGML_LOG_DEBUG(GGML_CUDA_NAME " pool[%d]: memory freed, %d captured CUDA graphs must be captured again\n", device, n_captured);+ }+ ctx.cuda_graphs.clear();+#endif // USE_CUDA_GRAPH+ }+void * alloc(size_t size, size_t * actual_size) override {#ifdef DEBUG_CUDA_MALLOCint nnz = 0;@@ -538,6 +554,7 @@ struct ggml_cuda_pool_leg : public ggml_cuda_pool {device, look_ahead_size/1024.0/1024.0, cached_bytes/1024.0/1024.0);CUDA_CHECK(cudaDeviceSynchronize());clear_pool();+ invalidate_cuda_graphs();err = ggml_cuda_device_malloc(&ptr, look_ahead_size, device);if (err == cudaSuccess) {GGML_LOG_DEBUG(GGML_CUDA_NAME " pool[%d]: retry succeeded\n", device);@@ -566,6 +583,7 @@ struct ggml_cuda_pool_leg : public ggml_cuda_pool {ggml_cuda_set_device(device);CUDA_CHECK(cudaFree(ptr));pool_size -= size;+ invalidate_cuda_graphs();}};@@ -720,14 +738,15 @@ struct ggml_cuda_pool_vmm : public ggml_cuda_pool {};#endif // defined(GGML_USE_VMM)-std::unique_ptr<ggml_cuda_pool> ggml_backend_cuda_context::new_pool_for_device(int device,- [[maybe_unused]] int stream_no) {+std::unique_ptr<ggml_cuda_pool> ggml_backend_cuda_context::new_pool_for_device(ggml_backend_cuda_context & ctx,+ int device,+ [[maybe_unused]] int stream_no) {#if defined(GGML_USE_VMM)if (ggml_cuda_info().devices[device].vmm) {return std::unique_ptr<ggml_cuda_pool>(new ggml_cuda_pool_vmm(device));}#endif // defined(GGML_USE_VMM)- return std::unique_ptr<ggml_cuda_pool>(new ggml_cuda_pool_leg(device));+ return std::unique_ptr<ggml_cuda_pool>(new ggml_cuda_pool_leg(ctx, device));}// destroying a cuBLAS handle while a graph is being captured in a different thread can result in a CUDA error