The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Improved memory management. (#5450)
comfyanonymous79d5ceae6efe
diff --git a/comfy/model_management.py b/comfy/model_management.pyindex a793cab3e..2a5009565 100644--- a/comfy/model_management.py+++ b/comfy/model_management.py@@ -23,6 +23,8 @@ from comfy.cli_args import argsimport torchimport sysimport platform+import weakref+import gcclass VRAMState(Enum):DISABLED = 0 #No vram present: no need to move models to vram@@ -287,11 +289,27 @@ def module_size(module):class LoadedModel:def __init__(self, model):- self.model = model+ self._set_model(model)self.device = model.load_device- self.weights_loaded = Falseself.real_model = Noneself.currently_used = True+ self.model_finalizer = None+ self._patcher_finalizer = None++ def _set_model(self, model):+ self._model = weakref.ref(model)+ if model.parent is not None:+ self._parent_model = weakref.ref(model.parent)+ self._patcher_finalizer = weakref.finalize(model, self._switch_parent)++ def _switch_parent(self):+ model = self._parent_model()+ if model is not None:+ self._set_model(model)++ @property+ def model(self):+ return self._model()def model_memory(self):return self.model.model_size()@@ -306,32 +324,23 @@ class LoadedModel:return self.model_memory()def model_load(self, lowvram_model_memory=0, force_patch_weights=False):- patch_model_to = self.device-self.model.model_patches_to(self.device)self.model.model_patches_to(self.model.model_dtype())- load_weights = not self.weights_loaded+ # if self.model.loaded_size() > 0:+ use_more_vram = lowvram_model_memory+ if use_more_vram == 0:+ use_more_vram = 1e32+ self.model_use_more_vram(use_more_vram, force_patch_weights=force_patch_weights)+ real_model = self.model.model- if self.model.loaded_size() > 0:- use_more_vram = lowvram_model_memory- if use_more_vram == 0:- use_more_vram = 1e32- self.model_use_more_vram(use_more_vram)- else:- try:- self.real_model = self.model.patch_model(device_to=patch_model_to, lowvram_model_memory=lowvram_model_memory, load_weights=load_weights, force_patch_weights=force_patch_weights)- except Exception as e:- self.model.unpatch_model(self.model.offload_device)- self.model_unload()- raise e-- if is_intel_xpu() and not args.disable_ipex_optimize and 'ipex' in globals() and self.real_model is not None:+ if is_intel_xpu() and not args.disable_ipex_optimize and 'ipex' in globals() and real_model is not None:with torch.no_grad():- self.real_model = ipex.optimize(self.real_model.eval(), inplace=True, graph_mode=True, concat_linear=True)+ real_model = ipex.optimize(real_model.eval(), inplace=True, graph_mode=True, concat_linear=True)- self.weights_loaded = True- return self.real_model+ self.real_model = weakref.ref(real_model)+ self.model_finalizer = weakref.finalize(real_model, cleanup_models)+ return real_modeldef should_reload_model(self, force_patch_weights=False):if force_patch_weights and self.model.lowvram_patch_counter() > 0:@@ -344,18 +353,23 @@ class LoadedModel:freed = self.model.partially_unload(self.model.offload_device, memory_to_free)if freed >= memory_to_free:return False- self.model.unpatch_model(self.model.offload_device, unpatch_weights=unpatch_weights)- self.model.model_patches_to(self.model.offload_device)- self.weights_loaded = self.weights_loaded and not unpatch_weights+ self.model.detach(unpatch_weights)+ self.model_finalizer.detach()+ self.model_finalizer = Noneself.real_model = Nonereturn True- def model_use_more_vram(self, extra_memory):- return self.model.partially_load(self.device, extra_memory)+ def model_use_more_vram(self, extra_memory, force_patch_weights=False):+ return self.model.partially_load(self.device, extra_memory, force_patch_weights=force_patch_weights)def __eq__(self, other):return self.model is other.model+ def __del__(self):+ if self._patcher_finalizer is not None:+ self._patcher_finalizer.detach()++def use_more_memory(extra_memory, loaded_models, device):for m in loaded_models:if m.device == device:@@ -386,38 +400,8 @@ def extra_reserved_memory():def minimum_inference_memory():return (1024 * 1024 * 1024) * 0.8 + extra_reserved_memory()-def unload_model_clones(model, unload_weights_only=True, force_unload=True):- to_unload = []- for i in range(len(current_loaded_models)):- if model.is_clone(current_loaded_models[i].model):- to_unload = [i] + to_unload-- if len(to_unload) == 0:- return True-- same_weights = 0- for i in to_unload:- if model.clone_has_same_weights(current_loaded_models[i].model):- same_weights += 1-- if same_weights == len(to_unload):- unload_weight = False- else:- unload_weight = True-- if not force_unload:- if unload_weights_only and unload_weight == False:- return None- else:- unload_weight = True-- for i in to_unload:- logging.debug("unload clone {} {}".format(i, unload_weight))- current_loaded_models.pop(i).model_unload(unpatch_weights=unload_weight)-- return unload_weight-def free_memory(memory_required, device, keep_loaded=[]):+ cleanup_models_gc()unloaded_model = []can_unload = []unloaded_models = []@@ -454,6 +438,7 @@ def free_memory(memory_required, device, keep_loaded=[]):return unloaded_modelsdef load_models_gpu(models, memory_required=0, force_patch_weights=False, minimum_memory_required=None, force_full_load=False):+ cleanup_models_gc()global vram_stateinference_memory = minimum_inference_memory()@@ -466,11 +451,9 @@ def load_models_gpu(models, memory_required=0, force_patch_weights=False, minimumodels = set(models)models_to_load = []- models_already_loaded = []+for x in models:loaded_model = LoadedModel(x)- loaded = None-try:loaded_model_index = current_loaded_models.index(loaded_model)except:@@ -478,51 +461,35 @@ def load_models_gpu(models, memory_required=0, force_patch_weights=False, minimuif loaded_model_index is not None:loaded = current_loaded_models[loaded_model_index]- if loaded.should_reload_model(force_patch_weights=force_patch_weights): #TODO: cleanup this model reload logic- current_loaded_models.pop(loaded_model_index).model_unload(unpatch_weights=True)- loaded = None- else:- loaded.currently_used = True- models_already_loaded.append(loaded)-- if loaded is None:+ loaded.currently_used = True+ models_to_load.append(loaded)+ else:if hasattr(x, "model"):logging.info(f"Requested to load {x.model.__class__.__name__}")models_to_load.append(loaded_model)- if len(models_to_load) == 0:- devs = set(map(lambda a: a.device, models_already_loaded))- for d in devs:- if d != torch.device("cpu"):- free_memory(extra_mem + offloaded_memory(models_already_loaded, d), d, models_already_loaded)- free_mem = get_free_memory(d)- if free_mem < minimum_memory_required:- logging.info("Unloading models for lowram load.") #TODO: partial model unloading when this case happens, also handle the opposite case where models can be unlowvramed.- models_to_load = free_memory(minimum_memory_required, d)- logging.info("{} models unloaded.".format(len(models_to_load)))- else:- use_more_memory(free_mem - minimum_memory_required, models_already_loaded, d)- if len(models_to_load) == 0:- return-- logging.info(f"Loading {len(models_to_load)} new model{'s' if len(models_to_load) > 1 else ''}")+ for loaded_model in models_to_load:+ to_unload = []+ for i in range(len(current_loaded_models)):+ if loaded_model.model.is_clone(current_loaded_models[i].model):+ to_unload = [i] + to_unload+ for i in to_unload:+ current_loaded_models.pop(i).model.detach(unpatch_all=False)total_memory_required = {}for loaded_model in models_to_load:- unload_model_clones(loaded_model.model, unload_weights_only=True, force_unload=False) #unload clones where the weights are differenttotal_memory_required[loaded_model.device] = total_memory_required.get(loaded_model.device, 0) + loaded_model.model_memory_required(loaded_model.device)- for loaded_model in models_already_loaded:- total_memory_required[loaded_model.device] = total_memory_required.get(loaded_model.device, 0) + loaded_model.model_memory_required(loaded_model.device)-- for loaded_model in models_to_load:- weights_unloaded = unload_model_clones(loaded_model.model, unload_weights_only=False, force_unload=False) #unload the rest of the clones where the weights can stay loaded- if weights_unloaded is not None:- loaded_model.weights_loaded = not weights_unloaded+ for device in total_memory_required:+ if device != torch.device("cpu"):+ free_memory(total_memory_required[device] * 1.1 + extra_mem, device)for device in total_memory_required:if device != torch.device("cpu"):- free_memory(total_memory_required[device] * 1.1 + extra_mem, device, models_already_loaded)+ free_mem = get_free_memory(device)+ if free_mem < minimum_memory_required:+ models_l = free_memory(minimum_memory_required, device)+ logging.info("{} models unloaded.".format(len(models_l)))for loaded_model in models_to_load:model = loaded_model.model@@ -544,17 +511,8 @@ def load_models_gpu(models, memory_required=0, force_patch_weights=False, minimucur_loaded_model = loaded_model.model_load(lowvram_model_memory, force_patch_weights=force_patch_weights)current_loaded_models.insert(0, loaded_model)--- devs = set(map(lambda a: a.device, models_already_loaded))- for d in devs:- if d != torch.device("cpu"):- free_mem = get_free_memory(d)- if free_mem > minimum_memory_required:- use_more_memory(free_mem - minimum_memory_required, models_already_loaded, d)return-def load_model_gpu(model):return load_models_gpu([model])@@ -568,21 +526,35 @@ def loaded_models(only_currently_used=False):output.append(m.model)return output-def cleanup_models(keep_clone_weights_loaded=False):++def cleanup_models_gc():+ do_gc = False+ for i in range(len(current_loaded_models)):+ cur = current_loaded_models[i]+ if cur.real_model() is not None and cur.model is None:+ logging.info("Potential memory leak detected with model {}, doing a full garbage collect, for maximum performance avoid circular references in the model code.".format(cur.real_model().__class__.__name__))+ do_gc = True+ break++ if do_gc:+ gc.collect()+ soft_empty_cache()++ for i in range(len(current_loaded_models)):+ cur = current_loaded_models[i]+ if cur.real_model() is not None and cur.model is None:+ logging.warning("WARNING, memory leak with model {}. Please make sure it is not being referenced from somewhere.".format(cur.real_model().__class__.__name__))++++def cleanup_models():to_delete = []for i in range(len(current_loaded_models)):- #TODO: very fragile function needs improvement- num_refs = sys.getrefcount(current_loaded_models[i].model)- if num_refs <= 2:- if not keep_clone_weights_loaded:- to_delete = [i] + to_delete- #TODO: find a less fragile way to do this.- elif sys.getrefcount(current_loaded_models[i].real_model) <= 3: #references from .real_model + the .model- to_delete = [i] + to_delete+ if current_loaded_models[i].real_model() is None:+ to_delete = [i] + to_deletefor i in to_delete:x = current_loaded_models.pop(i)- x.model_unload()del xdef dtype_size(dtype):diff --git a/comfy/model_patcher.py b/comfy/model_patcher.pyindex f53f10749..b64c795aa 100644--- a/comfy/model_patcher.py+++ b/comfy/model_patcher.py@@ -139,6 +139,7 @@ class ModelPatcher:self.offload_device = offload_deviceself.weight_inplace_update = weight_inplace_updateself.patches_uuid = uuid.uuid4()+ self.parent = Noneif not hasattr(self.model, 'model_loaded_weight_memory'):self.model.model_loaded_weight_memory = 0@@ -149,6 +150,9 @@ class ModelPatcher:if not hasattr(self.model, 'model_lowvram'):self.model.model_lowvram = False+ if not hasattr(self.model, 'current_weight_patches_uuid'):+ self.model.current_weight_patches_uuid = None+def model_size(self):if self.size > 0:return self.size@@ -172,6 +176,7 @@ class ModelPatcher:n.model_options = copy.deepcopy(self.model_options)n.backup = self.backupn.object_patches_backup = self.object_patches_backup+ n.parent = selfreturn ndef is_clone(self, other):@@ -464,6 +469,7 @@ class ModelPatcher:self.model.lowvram_patch_counter += patch_counterself.model.device = device_toself.model.model_loaded_weight_memory = mem_counter+ self.model.current_weight_patches_uuid = self.patches_uuiddef patch_model(self, device_to=None, lowvram_model_memory=0, load_weights=True, force_patch_weights=False):for k in self.object_patches:@@ -498,6 +504,7 @@ class ModelPatcher:else:comfy.utils.set_attr_param(self.model, k, bk.weight)+ self.model.current_weight_patches_uuid = Noneself.backup.clear()if device_to is not None:@@ -568,21 +575,42 @@ class ModelPatcher:self.model.model_loaded_weight_memory -= memory_freedreturn memory_freed- def partially_load(self, device_to, extra_memory=0):- self.unpatch_model(unpatch_weights=False)+ def partially_load(self, device_to, extra_memory=0, force_patch_weights=False):+ unpatch_weights = self.model.current_weight_patches_uuid is not None and (self.model.current_weight_patches_uuid != self.patches_uuid or force_patch_weights)+ # TODO: force_patch_weights should not unload + reload full model+ used = self.model.model_loaded_weight_memory+ self.unpatch_model(self.offload_device, unpatch_weights=unpatch_weights)+ if unpatch_weights:+ extra_memory += (used - self.model.model_loaded_weight_memory)+self.patch_model(load_weights=False)full_load = False- if self.model.model_lowvram == False:+ if self.model.model_lowvram == False and self.model.model_loaded_weight_memory > 0:return 0if self.model.model_loaded_weight_memory + extra_memory > self.model_size():full_load = Truecurrent_used = self.model.model_loaded_weight_memory- self.load(device_to, lowvram_model_memory=current_used + extra_memory, full_load=full_load)+ try:+ self.load(device_to, lowvram_model_memory=current_used + extra_memory, force_patch_weights=force_patch_weights, full_load=full_load)+ except Exception as e:+ self.detach()+ raise e+return self.model.model_loaded_weight_memory - current_used+ def detach(self, unpatch_all=True):+ self.model_patches_to(self.offload_device)+ if unpatch_all:+ self.unpatch_model(self.offload_device, unpatch_weights=unpatch_all)+ return self.model+def current_loaded_device(self):return self.model.devicedef calculate_weight(self, patches, weight, key, intermediate_dtype=torch.float32):print("WARNING the ModelPatcher.calculate_weight function is deprecated, please use: comfy.lora.calculate_weight instead")return comfy.lora.calculate_weight(patches, weight, key, intermediate_dtype=intermediate_dtype)++ def __del__(self):+ self.detach(unpatch_all=False)+diff --git a/execution.py b/execution.pyindex 6c386341b..768e35abc 100644--- a/execution.py+++ b/execution.py@@ -480,7 +480,7 @@ class PromptExecutor:if self.caches.outputs.get(node_id) is not None:cached_nodes.append(node_id)- comfy.model_management.cleanup_models(keep_clone_weights_loaded=True)+ comfy.model_management.cleanup_models_gc()self.add_message("execution_cached",{ "nodes": cached_nodes, "prompt_id": prompt_id},broadcast=False)diff --git a/main.py b/main.pyindex 05eb31c7a..c2c2ff8c8 100644--- a/main.py+++ b/main.py@@ -154,7 +154,6 @@ def prompt_worker(q, server):if need_gc:current_time = time.perf_counter()if (current_time - last_gc_collect) > gc_collect_interval:- comfy.model_management.cleanup_models()gc.collect()comfy.model_management.soft_empty_cache()last_gc_collect = current_time