The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Integrate RAM cache with model RAM management (#13173)
rattusb353a7c863e0
diff --git a/comfy/cli_args.py b/comfy/cli_args.pyindex 13612175e..dbaadf723 100644--- a/comfy/cli_args.py+++ b/comfy/cli_args.py@@ -110,11 +110,13 @@ parser.add_argument("--preview-method", type=LatentPreviewMethod, default=Latentparser.add_argument("--preview-size", type=int, default=512, help="Sets the maximum preview size for sampler nodes.")+CACHE_RAM_AUTO_GB = -1.0+cache_group = parser.add_mutually_exclusive_group()cache_group.add_argument("--cache-classic", action="store_true", help="Use the old style (aggressive) caching.")cache_group.add_argument("--cache-lru", type=int, default=0, help="Use LRU caching with a maximum of N node results cached. May use more RAM/VRAM.")cache_group.add_argument("--cache-none", action="store_true", help="Reduced RAM/VRAM usage at the expense of executing every node for each run.")-cache_group.add_argument("--cache-ram", nargs='?', const=4.0, type=float, default=0, help="Use RAM pressure caching with the specified headroom threshold. If available RAM drops below the threhold the cache remove large items to free RAM. Default 4GB")+cache_group.add_argument("--cache-ram", nargs='?', const=CACHE_RAM_AUTO_GB, type=float, default=0, help="Use RAM pressure caching with the specified headroom threshold. If available RAM drops below the threshold the cache removes large items to free RAM. Default (when no value is provided): 25%% of system RAM (min 4GB, max 32GB).")attn_group = parser.add_mutually_exclusive_group()attn_group.add_argument("--use-split-cross-attention", action="store_true", help="Use the split cross attention optimization. Ignored when xformers is used.")diff --git a/comfy/memory_management.py b/comfy/memory_management.pyindex f9078fe7c..48e3c11da 100644--- a/comfy/memory_management.py+++ b/comfy/memory_management.py@@ -141,3 +141,17 @@ def interpret_gathered_like(tensors, gathered):return dest_viewsaimdo_enabled = False++extra_ram_release_callback = None+RAM_CACHE_HEADROOM = 0++def set_ram_cache_release_state(callback, headroom):+ global extra_ram_release_callback+ global RAM_CACHE_HEADROOM+ extra_ram_release_callback = callback+ RAM_CACHE_HEADROOM = max(0, int(headroom))++def extra_ram_release(target):+ if extra_ram_release_callback is None:+ return 0+ return extra_ram_release_callback(target)diff --git a/comfy/model_management.py b/comfy/model_management.pyindex 9617d8388..ce079cf2f 100644--- a/comfy/model_management.py+++ b/comfy/model_management.py@@ -669,7 +669,7 @@ def free_memory(memory_required, device, keep_loaded=[], for_dynamic=False, pinsfor i in range(len(current_loaded_models) -1, -1, -1):shift_model = current_loaded_models[i]- if shift_model.device == device:+ if device is None or shift_model.device == device:if shift_model not in keep_loaded and not shift_model.is_dead():can_unload.append((-shift_model.model_offloaded_memory(), sys.getrefcount(shift_model.model), shift_model.model_memory(), i))shift_model.currently_used = False@@ -679,8 +679,8 @@ def free_memory(memory_required, device, keep_loaded=[], for_dynamic=False, pinsi = x[-1]memory_to_free = 1e32pins_to_free = 1e32- if not DISABLE_SMART_MEMORY:- memory_to_free = memory_required - get_free_memory(device)+ if not DISABLE_SMART_MEMORY or device is None:+ memory_to_free = 0 if device is None else memory_required - get_free_memory(device)pins_to_free = pins_required - get_free_ram()if current_loaded_models[i].model.is_dynamic() and for_dynamic:#don't actually unload dynamic models for the sake of other dynamic models@@ -708,7 +708,7 @@ def free_memory(memory_required, device, keep_loaded=[], for_dynamic=False, pinsif len(unloaded_model) > 0:soft_empty_cache()- else:+ elif device is not None:if vram_state != VRAMState.HIGH_VRAM:mem_free_total, mem_free_torch = get_free_memory(device, torch_free_too=True)if mem_free_torch > mem_free_total * 0.25:diff --git a/comfy/model_patcher.py b/comfy/model_patcher.pyindex c26d37db2..6deb71e12 100644--- a/comfy/model_patcher.py+++ b/comfy/model_patcher.py@@ -300,9 +300,6 @@ class ModelPatcher:def model_mmap_residency(self, free=False):return comfy.model_management.module_mmap_residency(self.model, free=free)- def get_ram_usage(self):- return self.model_size()-def loaded_size(self):return self.model.model_loaded_weight_memorydiff --git a/comfy/pinned_memory.py b/comfy/pinned_memory.pyindex f6fb806c4..6f142282d 100644--- a/comfy/pinned_memory.py+++ b/comfy/pinned_memory.py@@ -2,6 +2,7 @@ import comfy.model_managementimport comfy.memory_managementimport comfy_aimdo.host_bufferimport comfy_aimdo.torch+import psutilfrom comfy.cli_args import args@@ -12,6 +13,11 @@ def pin_memory(module):if module.pin_failed or args.disable_pinned_memory or get_pin(module) is not None:return#FIXME: This is a RAM cache trigger event+ ram_headroom = comfy.memory_management.RAM_CACHE_HEADROOM+ #we split the difference and assume half the RAM cache headroom is for us+ if ram_headroom > 0 and psutil.virtual_memory().available < (ram_headroom * 0.5):+ comfy.memory_management.extra_ram_release(ram_headroom)+size = comfy.memory_management.vram_aligned_size([ module.weight, module.bias ])if comfy.model_management.MAX_PINNED_MEMORY <= 0 or (comfy.model_management.TOTAL_PINNED_MEMORY + size) > comfy.model_management.MAX_PINNED_MEMORY:diff --git a/comfy/sd.py b/comfy/sd.pyindex e2645438c..e1a2840d2 100644--- a/comfy/sd.py+++ b/comfy/sd.py@@ -280,9 +280,6 @@ class CLIP:n.apply_hooks_to_conds = self.apply_hooks_to_condsreturn n- def get_ram_usage(self):- return self.patcher.get_ram_usage()-def add_patches(self, patches, strength_patch=1.0, strength_model=1.0):return self.patcher.add_patches(patches, strength_patch, strength_model)@@ -840,9 +837,6 @@ class VAE:self.size = comfy.model_management.module_size(self.first_stage_model)return self.size- def get_ram_usage(self):- return self.model_size()-def throw_exception_if_invalid(self):if self.first_stage_model is None:raise RuntimeError("ERROR: VAE is invalid: None\n\nIf the VAE is from a checkpoint loader node your checkpoint does not contain a valid VAE.")diff --git a/comfy_execution/caching.py b/comfy_execution/caching.pyindex 78212bde3..f9c913bdb 100644--- a/comfy_execution/caching.py+++ b/comfy_execution/caching.py@@ -1,6 +1,5 @@import asyncioimport bisect-import gcimport itertoolsimport psutilimport time@@ -475,6 +474,10 @@ class LRUCache(BasicCache):self._mark_used(node_id)return await self._set_immediate(node_id, value)+ def set_local(self, node_id, value):+ self._mark_used(node_id)+ BasicCache.set_local(self, node_id, value)+async def ensure_subcache_for(self, node_id, children_ids):# Just uses subcaches for tracking 'live' nodesawait super()._ensure_subcache(node_id, children_ids)@@ -489,15 +492,10 @@ class LRUCache(BasicCache):return self-#Iterating the cache for usage analysis might be expensive, so if we trigger make sure-#to take a chunk out to give breathing space on high-node / low-ram-per-node flows.--RAM_CACHE_HYSTERESIS = 1.1+#Small baseline weight used when a cache entry has no measurable CPU tensors.+#Keeps unknown-sized entries in eviction scoring without dominating tensor-backed entries.-#This is kinda in GB but not really. It needs to be non-zero for the below heuristic-#and as long as Multi GB models dwarf this it will approximate OOM scoring OK--RAM_CACHE_DEFAULT_RAM_USAGE = 0.1+RAM_CACHE_DEFAULT_RAM_USAGE = 0.05#Exponential bias towards evicting older workflows so garbage will be taken out#in constantly changing setups.@@ -521,19 +519,17 @@ class RAMPressureCache(LRUCache):self.timestamps[self.cache_key_set.get_data_key(node_id)] = time.time()return await super().get(node_id)- def poll(self, ram_headroom):- def _ram_gb():- return psutil.virtual_memory().available / (1024**3)+ def set_local(self, node_id, value):+ self.timestamps[self.cache_key_set.get_data_key(node_id)] = time.time()+ super().set_local(node_id, value)- if _ram_gb() > ram_headroom:- return- gc.collect()- if _ram_gb() > ram_headroom:+ def ram_release(self, target):+ if psutil.virtual_memory().available >= target:returnclean_list = []- for key, (outputs, _), in self.cache.items():+ for key, cache_entry in self.cache.items():oom_score = RAM_CACHE_OLD_WORKFLOW_OOM_MULTIPLIER ** (self.generation - self.used_generation[key])ram_usage = RAM_CACHE_DEFAULT_RAM_USAGE@@ -542,22 +538,20 @@ class RAMPressureCache(LRUCache):if outputs is None:returnfor output in outputs:- if isinstance(output, list):+ if isinstance(output, (list, tuple)):scan_list_for_ram_usage(output)elif isinstance(output, torch.Tensor) and output.device.type == 'cpu':- #score Tensors at a 50% discount for RAM usage as they are likely to- #be high value intermediates- ram_usage += (output.numel() * output.element_size()) * 0.5- elif hasattr(output, "get_ram_usage"):- ram_usage += output.get_ram_usage()- scan_list_for_ram_usage(outputs)+ ram_usage += output.numel() * output.element_size()+ scan_list_for_ram_usage(cache_entry.outputs)oom_score *= ram_usage#In the case where we have no information on the node ram usage at all,#break OOM score ties on the last touch timestamp (pure LRU)bisect.insort(clean_list, (oom_score, self.timestamps[key], key))- while _ram_gb() < ram_headroom * RAM_CACHE_HYSTERESIS and clean_list:+ while psutil.virtual_memory().available < target and clean_list:_, _, key = clean_list.pop()del self.cache[key]- gc.collect()+ self.used_generation.pop(key, None)+ self.timestamps.pop(key, None)+ self.children.pop(key, None)diff --git a/execution.py b/execution.pyindex 43c3c648d..5e02dffb2 100644--- a/execution.py+++ b/execution.py@@ -724,6 +724,9 @@ class PromptExecutor:self.add_message("execution_start", { "prompt_id": prompt_id}, broadcast=False)self._notify_prompt_lifecycle("start", prompt_id)+ ram_headroom = int(self.cache_args["ram"] * (1024 ** 3))+ ram_release_callback = self.caches.outputs.ram_release if self.cache_type == CacheType.RAM_PRESSURE else None+ comfy.memory_management.set_ram_cache_release_state(ram_release_callback, ram_headroom)try:with torch.inference_mode():@@ -773,7 +776,10 @@ class PromptExecutor:execution_list.unstage_node_execution()else: # result == ExecutionResult.SUCCESS:execution_list.complete_node_execution()- self.caches.outputs.poll(ram_headroom=self.cache_args["ram"])++ if self.cache_type == CacheType.RAM_PRESSURE:+ comfy.model_management.free_memory(0, None, pins_required=ram_headroom, ram_required=ram_headroom)+ comfy.memory_management.extra_ram_release(ram_headroom)else:# Only execute when the while-loop ends without break# Send cached UI for intermediate output nodes that weren't executed@@ -801,6 +807,7 @@ class PromptExecutor:if comfy.model_management.DISABLE_SMART_MEMORY:comfy.model_management.unload_all_models()finally:+ comfy.memory_management.set_ram_cache_release_state(None, 0)self._notify_prompt_lifecycle("end", prompt_id)diff --git a/main.py b/main.pyindex 058e8e2de..12b04719d 100644--- a/main.py+++ b/main.py@@ -275,15 +275,19 @@ def _collect_output_absolute_paths(history_result: dict) -> list[str]:def prompt_worker(q, server_instance):current_time: float = 0.0+ cache_ram = args.cache_ram+ if cache_ram < 0:+ cache_ram = min(32.0, max(4.0, comfy.model_management.total_ram * 0.25 / 1024.0))+cache_type = execution.CacheType.CLASSICif args.cache_lru > 0:cache_type = execution.CacheType.LRU- elif args.cache_ram > 0:+ elif cache_ram > 0:cache_type = execution.CacheType.RAM_PRESSUREelif args.cache_none:cache_type = execution.CacheType.NONE- e = execution.PromptExecutor(server_instance, cache_type=cache_type, cache_args={ "lru" : args.cache_lru, "ram" : args.cache_ram } )+ e = execution.PromptExecutor(server_instance, cache_type=cache_type, cache_args={ "lru" : args.cache_lru, "ram" : cache_ram } )last_gc_collect = 0need_gc = Falsegc_collect_interval = 10.0