The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Add RAM Pressure cache mode (#10454)
rattus513b0c46fba3
diff --git a/comfy/cli_args.py b/comfy/cli_args.pyindex 001abd843..3d5bc7c90 100644--- a/comfy/cli_args.py+++ b/comfy/cli_args.py@@ -105,6 +105,7 @@ cache_group = parser.add_mutually_exclusive_group()cache_group.add_argument("--cache-classic", action="store_true", help="Use the old style (aggressive) caching.")cache_group.add_argument("--cache-lru", type=int, default=0, help="Use LRU caching with a maximum of N node results cached. May use more RAM/VRAM.")cache_group.add_argument("--cache-none", action="store_true", help="Reduced RAM/VRAM usage at the expense of executing every node for each run.")+cache_group.add_argument("--cache-ram", nargs='?', const=4.0, type=float, default=0, help="Use RAM pressure caching with the specified headroom threshold. If available RAM drops below the threhold the cache remove large items to free RAM. Default 4GB")attn_group = parser.add_mutually_exclusive_group()attn_group.add_argument("--use-split-cross-attention", action="store_true", help="Use the split cross attention optimization. Ignored when xformers is used.")diff --git a/comfy/model_patcher.py b/comfy/model_patcher.pyindex ed3f3f5cb..674a214ca 100644--- a/comfy/model_patcher.py+++ b/comfy/model_patcher.py@@ -276,6 +276,9 @@ class ModelPatcher:self.size = comfy.model_management.module_size(self.model)return self.size+ def get_ram_usage(self):+ return self.model_size()+def loaded_size(self):return self.model.model_loaded_weight_memorydiff --git a/comfy/sd.py b/comfy/sd.pyindex de4eee96e..9e5ebbf15 100644--- a/comfy/sd.py+++ b/comfy/sd.py@@ -143,6 +143,9 @@ class CLIP:n.apply_hooks_to_conds = self.apply_hooks_to_condsreturn n+ def get_ram_usage(self):+ return self.patcher.get_ram_usage()+def add_patches(self, patches, strength_patch=1.0, strength_model=1.0):return self.patcher.add_patches(patches, strength_patch, strength_model)@@ -293,6 +296,7 @@ class VAE:self.working_dtypes = [torch.bfloat16, torch.float32]self.disable_offload = Falseself.not_video = False+ self.size = Noneself.downscale_index_formula = Noneself.upscale_index_formula = None@@ -595,6 +599,16 @@ class VAE:self.patcher = comfy.model_patcher.ModelPatcher(self.first_stage_model, load_device=self.device, offload_device=offload_device)logging.info("VAE load device: {}, offload device: {}, dtype: {}".format(self.device, offload_device, self.vae_dtype))+ self.model_size()++ def model_size(self):+ if self.size is not None:+ return self.size+ self.size = comfy.model_management.module_size(self.first_stage_model)+ return self.size++ def get_ram_usage(self):+ return self.model_size()def throw_exception_if_invalid(self):if self.first_stage_model is None:diff --git a/comfy_execution/caching.py b/comfy_execution/caching.pyindex 566bc3f9c..b498f43e7 100644--- a/comfy_execution/caching.py+++ b/comfy_execution/caching.py@@ -1,4 +1,9 @@+import bisect+import gcimport itertools+import psutil+import time+import torchfrom typing import Sequence, Mapping, Dictfrom comfy_execution.graph import DynamicPromptfrom abc import ABC, abstractmethod@@ -188,6 +193,9 @@ class BasicCache:self._clean_cache()self._clean_subcaches()+ def poll(self, **kwargs):+ pass+def _set_immediate(self, node_id, value):assert self.initializedcache_key = self.cache_key_set.get_data_key(node_id)@@ -276,6 +284,9 @@ class NullCache:def clean_unused(self):pass+ def poll(self, **kwargs):+ pass+def get(self, node_id):return None@@ -336,3 +347,75 @@ class LRUCache(BasicCache):self._mark_used(child_id)self.children[cache_key].append(self.cache_key_set.get_data_key(child_id))return self+++#Iterating the cache for usage analysis might be expensive, so if we trigger make sure+#to take a chunk out to give breathing space on high-node / low-ram-per-node flows.++RAM_CACHE_HYSTERESIS = 1.1++#This is kinda in GB but not really. It needs to be non-zero for the below heuristic+#and as long as Multi GB models dwarf this it will approximate OOM scoring OK++RAM_CACHE_DEFAULT_RAM_USAGE = 0.1++#Exponential bias towards evicting older workflows so garbage will be taken out+#in constantly changing setups.++RAM_CACHE_OLD_WORKFLOW_OOM_MULTIPLIER = 1.3++class RAMPressureCache(LRUCache):++ def __init__(self, key_class):+ super().__init__(key_class, 0)+ self.timestamps = {}++ def clean_unused(self):+ self._clean_subcaches()++ def set(self, node_id, value):+ self.timestamps[self.cache_key_set.get_data_key(node_id)] = time.time()+ super().set(node_id, value)++ def get(self, node_id):+ self.timestamps[self.cache_key_set.get_data_key(node_id)] = time.time()+ return super().get(node_id)++ def poll(self, ram_headroom):+ def _ram_gb():+ return psutil.virtual_memory().available / (1024**3)++ if _ram_gb() > ram_headroom:+ return+ gc.collect()+ if _ram_gb() > ram_headroom:+ return++ clean_list = []++ for key, (outputs, _), in self.cache.items():+ oom_score = RAM_CACHE_OLD_WORKFLOW_OOM_MULTIPLIER ** (self.generation - self.used_generation[key])++ ram_usage = RAM_CACHE_DEFAULT_RAM_USAGE+ def scan_list_for_ram_usage(outputs):+ nonlocal ram_usage+ for output in outputs:+ if isinstance(output, list):+ scan_list_for_ram_usage(output)+ elif isinstance(output, torch.Tensor) and output.device.type == 'cpu':+ #score Tensors at a 50% discount for RAM usage as they are likely to+ #be high value intermediates+ ram_usage += (output.numel() * output.element_size()) * 0.5+ elif hasattr(output, "get_ram_usage"):+ ram_usage += output.get_ram_usage()+ scan_list_for_ram_usage(outputs)++ oom_score *= ram_usage+ #In the case where we have no information on the node ram usage at all,+ #break OOM score ties on the last touch timestamp (pure LRU)+ bisect.insort(clean_list, (oom_score, self.timestamps[key], key))++ while _ram_gb() < ram_headroom * RAM_CACHE_HYSTERESIS and clean_list:+ _, _, key = clean_list.pop()+ del self.cache[key]+ gc.collect()diff --git a/comfy_execution/graph.py b/comfy_execution/graph.pyindex 341c9735d..0d811e354 100644--- a/comfy_execution/graph.py+++ b/comfy_execution/graph.py@@ -209,10 +209,15 @@ class ExecutionList(TopologicalSort):self.execution_cache_listeners[from_node_id] = set()self.execution_cache_listeners[from_node_id].add(to_node_id)- def get_output_cache(self, from_node_id, to_node_id):+ def get_cache(self, from_node_id, to_node_id):if not to_node_id in self.execution_cache:return None- return self.execution_cache[to_node_id].get(from_node_id)+ value = self.execution_cache[to_node_id].get(from_node_id)+ if value is None:+ return None+ #Write back to the main cache on touch.+ self.output_cache.set(from_node_id, value)+ return valuedef cache_update(self, node_id, value):if node_id in self.execution_cache_listeners:diff --git a/execution.py b/execution.pyindex 20e106213..17c77beab 100644--- a/execution.py+++ b/execution.py@@ -21,6 +21,7 @@ from comfy_execution.caching import (NullCache,HierarchicalCache,LRUCache,+ RAMPressureCache,)from comfy_execution.graph import (DynamicPrompt,@@ -88,49 +89,56 @@ class IsChangedCache:return self.is_changed[node_id]+class CacheEntry(NamedTuple):+ ui: dict+ outputs: list++class CacheType(Enum):CLASSIC = 0LRU = 1NONE = 2+ RAM_PRESSURE = 3class CacheSet:- def __init__(self, cache_type=None, cache_size=None):+ def __init__(self, cache_type=None, cache_args={}):if cache_type == CacheType.NONE:self.init_null_cache()logging.info("Disabling intermediate node cache.")+ elif cache_type == CacheType.RAM_PRESSURE:+ cache_ram = cache_args.get("ram", 16.0)+ self.init_ram_cache(cache_ram)+ logging.info("Using RAM pressure cache.")elif cache_type == CacheType.LRU:- if cache_size is None:- cache_size = 0+ cache_size = cache_args.get("lru", 0)self.init_lru_cache(cache_size)logging.info("Using LRU cache")else:self.init_classic_cache()- self.all = [self.outputs, self.ui, self.objects]+ self.all = [self.outputs, self.objects]# Performs like the old cache -- dump data ASAPdef init_classic_cache(self):self.outputs = HierarchicalCache(CacheKeySetInputSignature)- self.ui = HierarchicalCache(CacheKeySetInputSignature)self.objects = HierarchicalCache(CacheKeySetID)def init_lru_cache(self, cache_size):self.outputs = LRUCache(CacheKeySetInputSignature, max_size=cache_size)- self.ui = LRUCache(CacheKeySetInputSignature, max_size=cache_size)+ self.objects = HierarchicalCache(CacheKeySetID)++ def init_ram_cache(self, min_headroom):+ self.outputs = RAMPressureCache(CacheKeySetInputSignature)self.objects = HierarchicalCache(CacheKeySetID)def init_null_cache(self):self.outputs = NullCache()- #The UI cache is expected to be iterable at the end of each workflow- #so it must cache at least a full workflow. Use Heirachical- self.ui = HierarchicalCache(CacheKeySetInputSignature)self.objects = NullCache()def recursive_debug_dump(self):result = {"outputs": self.outputs.recursive_debug_dump(),- "ui": self.ui.recursive_debug_dump(),}return result@@ -157,14 +165,14 @@ def get_input_data(inputs, class_def, unique_id, execution_list=None, dynprompt=if execution_list is None:mark_missing()continue # This might be a lazily-evaluated input- cached_output = execution_list.get_output_cache(input_unique_id, unique_id)- if cached_output is None:+ cached = execution_list.get_cache(input_unique_id, unique_id)+ if cached is None or cached.outputs is None:mark_missing()continue- if output_index >= len(cached_output):+ if output_index >= len(cached.outputs):mark_missing()continue- obj = cached_output[output_index]+ obj = cached.outputs[output_index]input_data_all[x] = objelif input_category is not None:input_data_all[x] = [input_data]@@ -393,7 +401,7 @@ def format_value(x):else:return str(x)-async def execute(server, dynprompt, caches, current_item, extra_data, executed, prompt_id, execution_list, pending_subgraph_results, pending_async_nodes):+async def execute(server, dynprompt, caches, current_item, extra_data, executed, prompt_id, execution_list, pending_subgraph_results, pending_async_nodes, ui_outputs):unique_id = current_itemreal_node_id = dynprompt.get_real_node_id(unique_id)display_node_id = dynprompt.get_display_node_id(unique_id)@@ -401,12 +409,15 @@ async def execute(server, dynprompt, caches, current_item, extra_data, executed,inputs = dynprompt.get_node(unique_id)['inputs']class_type = dynprompt.get_node(unique_id)['class_type']class_def = nodes.NODE_CLASS_MAPPINGS[class_type]- if caches.outputs.get(unique_id) is not None:+ cached = caches.outputs.get(unique_id)+ if cached is not None:if server.client_id is not None:- cached_output = caches.ui.get(unique_id) or {}- server.send_sync("executed", { "node": unique_id, "display_node": display_node_id, "output": cached_output.get("output",None), "prompt_id": prompt_id }, server.client_id)+ cached_ui = cached.ui or {}+ server.send_sync("executed", { "node": unique_id, "display_node": display_node_id, "output": cached_ui.get("output",None), "prompt_id": prompt_id }, server.client_id)+ if cached.ui is not None:+ ui_outputs[unique_id] = cached.uiget_progress_state().finish_progress(unique_id)- execution_list.cache_update(unique_id, caches.outputs.get(unique_id))+ execution_list.cache_update(unique_id, cached)return (ExecutionResult.SUCCESS, None, None)input_data_all = None@@ -436,8 +447,8 @@ async def execute(server, dynprompt, caches, current_item, extra_data, executed,for r in result:if is_link(r):source_node, source_output = r[0], r[1]- node_output = execution_list.get_output_cache(source_node, unique_id)[source_output]- for o in node_output:+ node_cached = execution_list.get_cache(source_node, unique_id)+ for o in node_cached.outputs[source_output]:resolved_output.append(o)else:@@ -507,7 +518,7 @@ async def execute(server, dynprompt, caches, current_item, extra_data, executed,asyncio.create_task(await_completion())return (ExecutionResult.PENDING, None, None)if len(output_ui) > 0:- caches.ui.set(unique_id, {+ ui_outputs[unique_id] = {"meta": {"node_id": unique_id,"display_node": display_node_id,@@ -515,7 +526,7 @@ async def execute(server, dynprompt, caches, current_item, extra_data, executed,"real_node_id": real_node_id,},"output": output_ui- })+ }if server.client_id is not None:server.send_sync("executed", { "node": unique_id, "display_node": display_node_id, "output": output_ui, "prompt_id": prompt_id }, server.client_id)if has_subgraph:@@ -554,8 +565,9 @@ async def execute(server, dynprompt, caches, current_item, extra_data, executed,pending_subgraph_results[unique_id] = cached_outputsreturn (ExecutionResult.PENDING, None, None)- caches.outputs.set(unique_id, output_data)- execution_list.cache_update(unique_id, output_data)+ cache_entry = CacheEntry(ui=ui_outputs.get(unique_id), outputs=output_data)+ execution_list.cache_update(unique_id, cache_entry)+ caches.outputs.set(unique_id, cache_entry)except comfy.model_management.InterruptProcessingException as iex:logging.info("Processing interrupted")@@ -600,14 +612,14 @@ async def execute(server, dynprompt, caches, current_item, extra_data, executed,return (ExecutionResult.SUCCESS, None, None)class PromptExecutor:- def __init__(self, server, cache_type=False, cache_size=None):- self.cache_size = cache_size+ def __init__(self, server, cache_type=False, cache_args=None):+ self.cache_args = cache_argsself.cache_type = cache_typeself.server = serverself.reset()def reset(self):- self.caches = CacheSet(cache_type=self.cache_type, cache_size=self.cache_size)+ self.caches = CacheSet(cache_type=self.cache_type, cache_args=self.cache_args)self.status_messages = []self.success = True@@ -682,6 +694,7 @@ class PromptExecutor:broadcast=False)pending_subgraph_results = {}pending_async_nodes = {} # TODO - Unify this with pending_subgraph_results+ ui_node_outputs = {}executed = set()execution_list = ExecutionList(dynamic_prompt, self.caches.outputs)current_outputs = self.caches.outputs.all_node_ids()@@ -695,7 +708,7 @@ class PromptExecutor:breakassert node_id is not None, "Node ID should not be None at this point"- result, error, ex = await execute(self.server, dynamic_prompt, self.caches, node_id, extra_data, executed, prompt_id, execution_list, pending_subgraph_results, pending_async_nodes)+ result, error, ex = await execute(self.server, dynamic_prompt, self.caches, node_id, extra_data, executed, prompt_id, execution_list, pending_subgraph_results, pending_async_nodes, ui_node_outputs)self.success = result != ExecutionResult.FAILUREif result == ExecutionResult.FAILURE:self.handle_execution_error(prompt_id, dynamic_prompt.original_prompt, current_outputs, executed, error, ex)@@ -704,18 +717,16 @@ class PromptExecutor:execution_list.unstage_node_execution()else: # result == ExecutionResult.SUCCESS:execution_list.complete_node_execution()+ self.caches.outputs.poll(ram_headroom=self.cache_args["ram"])else:# Only execute when the while-loop ends without breakself.add_message("execution_success", { "prompt_id": prompt_id }, broadcast=False)ui_outputs = {}meta_outputs = {}- all_node_ids = self.caches.ui.all_node_ids()- for node_id in all_node_ids:- ui_info = self.caches.ui.get(node_id)- if ui_info is not None:- ui_outputs[node_id] = ui_info["output"]- meta_outputs[node_id] = ui_info["meta"]+ for node_id, ui_info in ui_node_outputs.items():+ ui_outputs[node_id] = ui_info["output"]+ meta_outputs[node_id] = ui_info["meta"]self.history_result = {"outputs": ui_outputs,"meta": meta_outputs,diff --git a/main.py b/main.pyindex 8d466d2eb..e1b0f1620 100644--- a/main.py+++ b/main.py@@ -172,10 +172,12 @@ def prompt_worker(q, server_instance):cache_type = execution.CacheType.CLASSICif args.cache_lru > 0:cache_type = execution.CacheType.LRU+ elif args.cache_ram > 0:+ cache_type = execution.CacheType.RAM_PRESSUREelif args.cache_none:cache_type = execution.CacheType.NONE- e = execution.PromptExecutor(server_instance, cache_type=cache_type, cache_size=args.cache_lru)+ e = execution.PromptExecutor(server_instance, cache_type=cache_type, cache_args={ "lru" : args.cache_lru, "ram" : args.cache_ram } )last_gc_collect = 0need_gc = Falsegc_collect_interval = 10.0