The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Dynamic VRAM unloading fix (#12227)
rattusde9ada6a4147
diff --git a/comfy/model_management.py b/comfy/model_management.pyindex 2167f81bf..cd035f017 100644--- a/comfy/model_management.py+++ b/comfy/model_management.py@@ -19,7 +19,8 @@import psutilimport loggingfrom enum import Enum-from comfy.cli_args import args, PerformanceFeature+from comfy.cli_args import args, PerformanceFeature, enables_dynamic_vram+import threadingimport torchimport sysimport platform@@ -650,7 +651,7 @@ def free_memory(memory_required, device, keep_loaded=[], for_dynamic=False, ram_soft_empty_cache()return unloaded_models-def load_models_gpu(models, memory_required=0, force_patch_weights=False, minimum_memory_required=None, force_full_load=False):+def load_models_gpu_orig(models, memory_required=0, force_patch_weights=False, minimum_memory_required=None, force_full_load=False):cleanup_models_gc()global vram_state@@ -746,8 +747,25 @@ def load_models_gpu(models, memory_required=0, force_patch_weights=False, minimucurrent_loaded_models.insert(0, loaded_model)return-def load_model_gpu(model):- return load_models_gpu([model])+def load_models_gpu_thread(models, memory_required, force_patch_weights, minimum_memory_required, force_full_load):+ with torch.inference_mode():+ load_models_gpu_orig(models, memory_required, force_patch_weights, minimum_memory_required, force_full_load)+ soft_empty_cache()++def load_models_gpu(models, memory_required=0, force_patch_weights=False, minimum_memory_required=None, force_full_load=False):+ #Deliberately load models outside of the Aimdo mempool so they can be retained accross+ #nodes. Use a dummy thread to do it as pytorch documents that mempool contexts are+ #thread local. So exploit that to escape context+ if enables_dynamic_vram():+ t = threading.Thread(+ target=load_models_gpu_thread,+ args=(models, memory_required, force_patch_weights, minimum_memory_required, force_full_load)+ )+ t.start()+ t.join()+ else:+ load_models_gpu_orig(models, memory_required=memory_required, force_patch_weights=force_patch_weights,+ minimum_memory_required=minimum_memory_required, force_full_load=force_full_load)def loaded_models(only_currently_used=False):output = []@@ -1717,9 +1735,6 @@ def debug_memory_summary():return torch.cuda.memory.memory_summary()return ""-#TODO: might be cleaner to put this somewhere else-import threading-class InterruptProcessingException(Exception):passdiff --git a/comfy/model_patcher.py b/comfy/model_patcher.pyindex b70c031bf..cdf289395 100644--- a/comfy/model_patcher.py+++ b/comfy/model_patcher.py@@ -1597,7 +1597,7 @@ class ModelPatcherDynamic(ModelPatcher):if unpatch_weights:self.partially_unload_ram(1e32)- self.partially_unload(None)+ self.partially_unload(None, 1e32)def partially_load(self, device_to, extra_memory=0, force_patch_weights=False):assert not force_patch_weights #See above