Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Print memory summary on OOM to help with debugging. (#11613)

comfyanonymous53e762a3af95
diff --git a/comfy/model_management.py b/comfy/model_management.py
index 87baedd73..2501cecb7 100644
--- a/comfy/model_management.py
+++ b/comfy/model_management.py
@@ -1542,6 +1542,10 @@ def soft_empty_cache(force=False):
def unload_all_models():
free_memory(1e30, get_torch_device())
+def debug_memory_summary():
+ if is_amd() or is_nvidia():
+ return torch.cuda.memory.memory_summary()
+ return ""
#TODO: might be cleaner to put this somewhere else
import threading
diff --git a/execution.py b/execution.py
index 38159b1f4..648f204ec 100644
--- a/execution.py
+++ b/execution.py
@@ -601,6 +601,7 @@ async def execute(server, dynprompt, caches, current_item, extra_data, executed,
if isinstance(ex, comfy.model_management.OOM_EXCEPTION):
tips = "This error means you ran out of memory on your GPU.\n\nTIPS: If the workflow worked before you might have accidentally set the batch_size to a large number."
+ logging.info("Memory summary: {}".format(comfy.model_management.debug_memory_summary()))
logging.error("Got an OOM, unloading all loaded models.")
comfy.model_management.unload_all_models()