LLM inference in C/C++
C++ ggml
cuda : add GGML_CUDA_GRAPH_DEBUG and read env vars once (#30327)
Georgi Gerganovf851e244d0dc
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuindex f4537d4479..9fe618f855 100644--- a/ggml/src/ggml-cuda/ggml-cuda.cu+++ b/ggml/src/ggml-cuda/ggml-cuda.cu@@ -138,10 +138,54 @@ int ggml_cuda_get_device() {return id;}+static bool ggml_cuda_env_flag(const char * name) {+ const char * env = getenv(name);+ return env != nullptr && std::atoi(env) != 0;+}++static int ggml_cuda_env_int(const char * name, int default_value) {+ const char * env = getenv(name);+ return env != nullptr ? std::atoi(env) : default_value;+}++static bool ggml_cuda_env_set(const char * name) {+ return getenv(name) != nullptr;+}++static bool ggml_cuda_unified_memory_enabled() {+ static const bool enabled = ggml_cuda_env_flag("GGML_CUDA_ENABLE_UNIFIED_MEMORY");+ return enabled;+}++static bool ggml_cuda_pinned_memory_disabled() {+ static const bool disabled = ggml_cuda_env_flag("GGML_CUDA_NO_PINNED");+ return disabled;+}++static bool ggml_cuda_register_host_enabled() {+ static const bool enabled = ggml_cuda_env_flag("GGML_CUDA_REGISTER_HOST");+ return enabled;+}++static bool ggml_cuda_fusion_disabled() {+ static const bool disabled = ggml_cuda_env_flag("GGML_CUDA_DISABLE_FUSION");+ return disabled;+}++static bool ggml_cuda_p2p_enabled() {+ static const bool enabled = ggml_cuda_env_set("GGML_CUDA_P2P");+ return enabled;+}++static int ggml_cuda_op_offload_min_batch() {+ static const int min_batch = ggml_cuda_env_int("GGML_OP_OFFLOAD_MIN_BATCH", 32);+ return min_batch;+}+static cudaError_t ggml_cuda_device_malloc(void ** ptr, size_t size, int device) {ggml_cuda_set_device(device);cudaError_t err;- if (getenv("GGML_CUDA_ENABLE_UNIFIED_MEMORY") != nullptr) {+ if (ggml_cuda_unified_memory_enabled()) {err = cudaMallocManaged(ptr, size);#if defined(GGML_USE_HIP)if (err == hipSuccess) {@@ -383,7 +427,7 @@ static ggml_cuda_device_info ggml_cuda_init() {// configure logging to stdout// CUBLAS_CHECK(cublasLoggerConfigure(1, 1, 0, nullptr));- if (getenv("GGML_CUDA_P2P") != nullptr) {+ if (ggml_cuda_p2p_enabled()) {for (int id = 0; id < info.physical_device_count; ++id) {CUDA_CHECK(cudaSetDevice(id));for (int id_other = 0; id_other < info.physical_device_count; ++id_other) {@@ -598,7 +642,7 @@ struct ggml_cuda_pool_vmm : public ggml_cuda_pool {CU_CHECK(cuMemRelease(handle));// VMM Bug fix for P2P access if GGML_CUDA_P2P is set, or if NCCL build- bool use_peer_access = getenv("GGML_CUDA_P2P") != nullptr;+ bool use_peer_access = ggml_cuda_p2p_enabled();#if defined(GGML_USE_NCCL)use_peer_access = true;#endif // defined(GGML_USE_NCCL)@@ -1272,7 +1316,7 @@ static void ggml_backend_cuda_host_buffer_free_buffer(ggml_backend_buffer_t buff}static void * ggml_cuda_host_malloc(size_t size) {- if (getenv("GGML_CUDA_NO_PINNED") != nullptr) {+ if (ggml_cuda_pinned_memory_disabled()) {return nullptr;}@@ -2641,6 +2685,11 @@ static bool ggml_cuda_is_view_or_noop(const ggml_tensor * t) {}#ifdef USE_CUDA_GRAPH+static bool ggml_cuda_graph_debug_enabled() {+ static const bool enabled = ggml_cuda_env_flag("GGML_CUDA_GRAPH_DEBUG");+ return enabled;+}+static bool ggml_cuda_graph_check_compability(ggml_cgraph * cgraph) {bool use_cuda_graph = true;@@ -2660,9 +2709,9 @@ static bool ggml_cuda_graph_check_compability(ggml_cgraph * cgraph) {// the mul_mat_id fallback path synchronizes the stream, so we cannot use CUDA graphs// ref: https://github.com/ggml-org/llama.cpp/pull/18958use_cuda_graph = false;-#ifndef NDEBUG- GGML_LOG_DEBUG("%s: disabling CUDA graphs due to unsupported node type\n", __func__);-#endif+ if (ggml_cuda_graph_debug_enabled()) {+ GGML_LOG_DEBUG("%s: disabling CUDA graphs due to unsupported node type\n", __func__);+ }}}@@ -2686,7 +2735,9 @@ static bool ggml_cuda_graph_update_required(ggml_backend_cuda_context * cuda_ctxif (cgraph->uid != 0 &&cgraph->uid == graph->uid) {- GGML_LOG_DEBUG("CUDA Graph id %zu reused\n", cgraph->uid);+ if (ggml_cuda_graph_debug_enabled()) {+ GGML_LOG_DEBUG("CUDA Graph id %zu reused\n", cgraph->uid);+ }GGML_ASSERT((int)graph->node_props.size() == cgraph->n_nodes);return false;}@@ -2733,9 +2784,9 @@ static void ggml_cuda_graph_update_executable(ggml_backend_cuda_context * cuda_c#endif // CUDART_VERSION >= 12000if (stat == cudaErrorGraphExecUpdateFailure) {-#ifndef NDEBUG- GGML_LOG_DEBUG("%s: CUDA graph update failed\n", __func__);-#endif+ if (ggml_cuda_graph_debug_enabled()) {+ GGML_LOG_DEBUG("%s: CUDA graph update failed\n", __func__);+ }// The pre-existing graph exec cannot be updated due to violated constraints// so instead clear error and re-instantiate@@ -3605,8 +3656,7 @@ static bool ggml_cuda_can_fuse(const struct ggml_cgraph * cgraph,// try and fuse nodes and return the number of nodes to skipstatic int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph * cgraph, int i) {- static bool disable_fusion = getenv("GGML_CUDA_DISABLE_FUSION") != nullptr && std::atoi(getenv("GGML_CUDA_DISABLE_FUSION"));- if (disable_fusion) {+ if (ggml_cuda_fusion_disabled()) {return 0;}@@ -4620,7 +4670,7 @@ static bool ggml_cuda_graph_set_enabled(ggml_backend_cuda_context * cuda_ctx, coif (graph->graph == nullptr) {if (ggml_cuda_info().devices[cuda_ctx->device].cc < GGML_CUDA_CC_VOLTA) {- if (!graph->disable_due_to_gpu_arch) {+ if (!graph->disable_due_to_gpu_arch && ggml_cuda_graph_debug_enabled()) {GGML_LOG_DEBUG("%s: disabling CUDA graphs due to GPU architecture\n", __func__);}graph->disable_due_to_gpu_arch = true;@@ -4655,7 +4705,9 @@ static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend,// Warmup: need at least 2 calls with no property change on the 2nd callif (!properties_changed) {graph->warmup_complete = true;- GGML_LOG_DEBUG("%s: CUDA graph warmup complete\n", __func__);+ if (ggml_cuda_graph_debug_enabled()) {+ GGML_LOG_DEBUG("%s: CUDA graph warmup complete\n", __func__);+ }use_cuda_graph = true;cuda_graph_update_required = true;}@@ -4665,7 +4717,9 @@ static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend,if (properties_changed) {// Properties changed - reset warmup, execute directly until stable againgraph->warmup_complete = false;- GGML_LOG_DEBUG("%s: CUDA graph warmup reset\n", __func__);+ if (ggml_cuda_graph_debug_enabled()) {+ GGML_LOG_DEBUG("%s: CUDA graph warmup reset\n", __func__);+ }} else {use_cuda_graph = true;cuda_graph_update_required = graph->instance == nullptr;@@ -4718,7 +4772,7 @@ static void ggml_backend_cuda_event_wait(ggml_backend_t backend, ggml_backend_evstatic void ggml_backend_cuda_graph_optimize(ggml_backend_t backend, ggml_cgraph * cgraph, ggml_backend_graph_optimize_params * params) {ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;- static const bool disable_fusion = getenv("GGML_CUDA_DISABLE_FUSION") != nullptr && std::atoi(getenv("GGML_CUDA_DISABLE_FUSION"));+ const bool disable_fusion = ggml_cuda_fusion_disabled();auto add_alloc_deps = [&](size_t start, size_t last_node) {@@ -5138,7 +5192,7 @@ void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * tot}bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size) {- if (getenv("GGML_CUDA_REGISTER_HOST") == nullptr) {+ if (!ggml_cuda_register_host_enabled()) {return false;}@@ -5161,7 +5215,7 @@ bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size) {}void ggml_backend_cuda_unregister_host_buffer(void * buffer) {- if (getenv("GGML_CUDA_REGISTER_HOST") == nullptr) {+ if (!ggml_cuda_register_host_enabled()) {return;}@@ -5288,8 +5342,7 @@ static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t *CUDA_CHECK(cudaGetDeviceProperties(&prop, ggml_cuda_get_physical_device(ctx->device)));// Check if UMA is explicitly enabled via environment variable- bool uma_env = getenv("GGML_CUDA_ENABLE_UNIFIED_MEMORY") != nullptr;- bool is_uma = prop.integrated > 0 || uma_env;+ bool is_uma = prop.integrated > 0 || ggml_cuda_unified_memory_enabled();if (is_uma) {// For UMA systems (like DGX Spark), use system memory info@@ -5330,7 +5383,7 @@ static void ggml_backend_cuda_device_get_props(ggml_backend_dev_t dev, ggml_backprops->device_id = ctx->pci_bus_id.empty() ? nullptr : ctx->pci_bus_id.c_str();ggml_backend_cuda_device_get_memory(dev, &props->memory_free, &props->memory_total);- bool host_buffer = getenv("GGML_CUDA_NO_PINNED") == nullptr;+ const bool host_buffer = !ggml_cuda_pinned_memory_disabled();#ifdef GGML_CUDA_NO_PEER_COPYbool events = false;#else@@ -6026,7 +6079,7 @@ ggml_backend_reg_t ggml_backend_cuda_reg() {std::lock_guard<std::mutex> lock(mutex);if (!initialized) {ggml_backend_cuda_reg_context * ctx = new ggml_backend_cuda_reg_context;- const int min_batch_size = getenv("GGML_OP_OFFLOAD_MIN_BATCH") ? atoi(getenv("GGML_OP_OFFLOAD_MIN_BATCH")) : 32;+ const int min_batch_size = ggml_cuda_op_offload_min_batch();const ggml_cuda_device_info & info = ggml_cuda_info();const bool virtual_devices = info.device_count > info.physical_device_count;