The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Try to fix the model reloading issue some people have. (#14822)
comfyanonymousb7a648ca2011
diff --git a/comfy/model_management.py b/comfy/model_management.pyindex b15d08ba1..222005b6f 100644--- a/comfy/model_management.py+++ b/comfy/model_management.py@@ -616,6 +616,8 @@ PIN_PRESSURE_HYSTERESIS = 256 * 1024 * 1024#Freeing registerables on pressure does imply a GPU sync, so go big on#the hysteresis so each expensive sync gives us back a good chunk.REGISTERABLE_PIN_HYSTERESIS = 2048 * 1024 * 1024+WINDOWS_PIN_EVICTION_SWAP_PERCENT = 5.0+WINDOWS_PIN_EVICTION_EMERGENCY_AVAILABLE = 512 * 1024 ** 2def module_size(module):module_mem = 0@@ -642,6 +644,15 @@ def free_pins(size, evict_active=False):size -= freedreturn freed_total+def should_free_pins_for_ram_pressure(shortfall):+ if shortfall <= 0:+ return False+ if not WINDOWS:+ return True+ if psutil.virtual_memory().available < WINDOWS_PIN_EVICTION_EMERGENCY_AVAILABLE:+ return True+ return psutil.swap_memory().percent >= WINDOWS_PIN_EVICTION_SWAP_PERCENT+def ensure_pin_budget(size, evict_active=False):if args.high_ram:return Truediff --git a/comfy_execution/caching.py b/comfy_execution/caching.pyindex ad75a0e50..6bd99b68f 100644--- a/comfy_execution/caching.py+++ b/comfy_execution/caching.py@@ -503,6 +503,8 @@ RAM_CACHE_DEFAULT_RAM_USAGE = 0.05RAM_CACHE_OLD_WORKFLOW_OOM_MULTIPLIER = 1.3+RAM_CACHE_LARGE_INTERMEDIATE = 512 * 1024 ** 2+def all_outputs_dynamic(outputs):if outputs is None:@@ -517,7 +519,6 @@ def all_outputs_dynamic(outputs):return True-class RAMPressureCache(LRUCache):def __init__(self, key_class, enable_providers=False):@@ -539,9 +540,9 @@ class RAMPressureCache(LRUCache):self.timestamps[self.cache_key_set.get_data_key(node_id)] = time.time()super().set_local(node_id, value)- def ram_release(self, target, free_active=False):+ def ram_release(self, target, free_active=False, min_entry_size=0):if psutil.virtual_memory().available >= target:- return+ return 0clean_list = []@@ -555,8 +556,9 @@ class RAMPressureCache(LRUCache):oom_score = RAM_CACHE_OLD_WORKFLOW_OOM_MULTIPLIER ** (self.generation - self.used_generation[key])ram_usage = RAM_CACHE_DEFAULT_RAM_USAGE+ oom_ram_usage = ram_usagedef scan_list_for_ram_usage(outputs):- nonlocal ram_usage+ nonlocal ram_usage, oom_ram_usageif outputs is None:returnfor output in outputs:@@ -564,19 +566,26 @@ class RAMPressureCache(LRUCache):scan_list_for_ram_usage(output)elif isinstance(output, torch.Tensor) and output.device.type == 'cpu':ram_usage += output.numel() * output.element_size()+ oom_ram_usage += output.numel() * output.element_size()elif isinstance(output, ModelPatcher) and self.used_generation[key] != self.generation:#old ModelPatchers are the first to go- ram_usage = 1e30+ oom_ram_usage = 1e30scan_list_for_ram_usage(cache_entry.outputs)- oom_score *= ram_usage+ if ram_usage < min_entry_size:+ continue++ oom_score *= oom_ram_usage#In the case where we have no information on the node ram usage at all,#break OOM score ties on the last touch timestamp (pure LRU)- bisect.insort(clean_list, (oom_score, self.timestamps[key], key))+ bisect.insort(clean_list, (oom_score, self.timestamps[key], key, ram_usage))+ freed = 0while psutil.virtual_memory().available < target and clean_list:- _, _, key = clean_list.pop()+ _, _, key, ram_usage = clean_list.pop()del self.cache[key]self.used_generation.pop(key, None)self.timestamps.pop(key, None)self.children.pop(key, None)+ freed += ram_usage+ return freeddiff --git a/execution.py b/execution.pyindex c45317593..19b8cdd68 100644--- a/execution.py+++ b/execution.py@@ -29,6 +29,7 @@ from comfy_execution.caching import (HierarchicalCache,LRUCache,RAMPressureCache,+ RAM_CACHE_LARGE_INTERMEDIATE,)from comfy_execution.graph import (DynamicPrompt,@@ -794,12 +795,16 @@ class PromptExecutor:if self.cache_type == CacheType.RAM_PRESSURE:ram_release_callback(ram_inactive_headroom)ram_shortfall = ram_headroom - psutil.virtual_memory().available- freed = comfy.model_management.free_pins(ram_shortfall + 512 * (1024 ** 2))- if freed < ram_shortfall:- if freed > 64 * (1024 ** 2):- # AIMDO MEM_DECOMMIT can outrun psutil.available catching up.- time.sleep(0.05)- ram_release_callback(ram_headroom, free_active=True)+ if ram_shortfall > 0:+ freed = ram_release_callback(ram_headroom, free_active=True, min_entry_size=RAM_CACHE_LARGE_INTERMEDIATE)+ ram_shortfall -= freed+ if comfy.model_management.should_free_pins_for_ram_pressure(ram_shortfall):+ freed = comfy.model_management.free_pins(ram_shortfall + 512 * (1024 ** 2))+ if freed < ram_shortfall:+ if freed > 64 * (1024 ** 2):+ # AIMDO MEM_DECOMMIT can outrun psutil.available catching up.+ time.sleep(0.05)+ ram_release_callback(ram_headroom, free_active=True)else:# Only execute when the while-loop ends without break# Send cached UI for intermediate output nodes that weren't executed