The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
ModelPatcherDynamic: Force load all non-comfy weights (#12739)
rattus09bcbddfcf80
diff --git a/comfy/model_management.py b/comfy/model_management.pyindex c817d43b5..0e0e96672 100644--- a/comfy/model_management.py+++ b/comfy/model_management.py@@ -32,9 +32,6 @@ import comfy.memory_managementimport comfy.utilsimport comfy.quant_ops-import comfy_aimdo.torch-import comfy_aimdo.model_vbar-class VRAMState(Enum):DISABLED = 0 #No vram present: no need to move models to vramNO_VRAM = 1 #Very low vram: enable all the options to save vram@@ -1206,43 +1203,6 @@ def cast_to_gathered(tensors, r, non_blocking=False, stream=None):def cast_to(weight, dtype=None, device=None, non_blocking=False, copy=False, stream=None, r=None):- if hasattr(weight, "_v"):- #Unexpected usage patterns. There is no reason these don't work but they- #have no testing and no callers do this.- assert r is None- assert stream is None-- cast_geometry = comfy.memory_management.tensors_to_geometries([ weight ])-- if dtype is None:- dtype = weight._model_dtype-- signature = comfy_aimdo.model_vbar.vbar_fault(weight._v)- if signature is not None:- if comfy_aimdo.model_vbar.vbar_signature_compare(signature, weight._v_signature):- v_tensor = weight._v_tensor- else:- raw_tensor = comfy_aimdo.torch.aimdo_to_tensor(weight._v, device)- v_tensor = comfy.memory_management.interpret_gathered_like(cast_geometry, raw_tensor)[0]- weight._v_tensor = v_tensor- weight._v_signature = signature- #Send it over- v_tensor.copy_(weight, non_blocking=non_blocking)- return v_tensor.to(dtype=dtype)-- r = torch.empty_like(weight, dtype=dtype, device=device)-- if weight.dtype != r.dtype and weight.dtype != weight._model_dtype:- #Offloaded casting could skip this, however it would make the quantizations- #inconsistent between loaded and offloaded weights. So force the double casting- #that would happen in regular flow to make offload deterministic.- cast_buffer = torch.empty_like(weight, dtype=weight._model_dtype, device=device)- cast_buffer.copy_(weight, non_blocking=non_blocking)- weight = cast_buffer- r.copy_(weight, non_blocking=non_blocking)-- return r-if device is None or weight.device == device:if not copy:if dtype is None or weight.dtype == dtype:diff --git a/comfy/model_patcher.py b/comfy/model_patcher.pyindex 3fc76d9db..e380e406b 100644--- a/comfy/model_patcher.py+++ b/comfy/model_patcher.py@@ -1435,10 +1435,6 @@ class ModelPatcherDynamic(ModelPatcher):def __init__(self, model, load_device, offload_device, size=0, weight_inplace_update=False):super().__init__(model, load_device, offload_device, size, weight_inplace_update)- #this is now way more dynamic and we dont support the same base model for both Dynamic- #and non-dynamic patchers.- if hasattr(self.model, "model_loaded_weight_memory"):- del self.model.model_loaded_weight_memoryif not hasattr(self.model, "dynamic_vbars"):self.model.dynamic_vbars = {}self.non_dynamic_delegate_model = None@@ -1461,9 +1457,7 @@ class ModelPatcherDynamic(ModelPatcher):def loaded_size(self):vbar = self._vbar_get()- if vbar is None:- return 0- return vbar.loaded_size()+ return (vbar.loaded_size() if vbar is not None else 0) + self.model.model_loaded_weight_memorydef get_free_memory(self, device):#NOTE: on high condition / batch counts, estimate should have already vacated@@ -1504,6 +1498,7 @@ class ModelPatcherDynamic(ModelPatcher):num_patches = 0allocated_size = 0+ self.model.model_loaded_weight_memory = 0with self.use_ejected():self.unpatch_hooks()@@ -1512,10 +1507,6 @@ class ModelPatcherDynamic(ModelPatcher):if vbar is not None:vbar.prioritize()- #We force reserve VRAM for the non comfy-weight so we dont have to deal- #with pin and unpin syncrhonization which can be expensive for small weights- #with a high layer rate (e.g. autoregressive LLMs).- #prioritize the non-comfy weights (note the order reverse).loading = self._load_list(prio_comfy_cast_weights=True, default_device=device_to)loading.sort(reverse=True)@@ -1558,6 +1549,9 @@ class ModelPatcherDynamic(ModelPatcher):if key in self.backup:comfy.utils.set_attr_param(self.model, key, self.backup[key].weight)self.patch_weight_to_device(key, device_to=device_to)+ weight, _, _ = get_key_weight(self.model, key)+ if weight is not None:+ self.model.model_loaded_weight_memory += weight.numel() * weight.element_size()if hasattr(m, "comfy_cast_weights"):m.comfy_cast_weights = True@@ -1583,21 +1577,15 @@ class ModelPatcherDynamic(ModelPatcher):for param in params:key = key_param_name_to_key(n, param)weight, _, _ = get_key_weight(self.model, key)- weight.seed_key = key- set_dirty(weight, dirty)- geometry = weight- model_dtype = getattr(m, param + "_comfy_model_dtype", None) or weight.dtype- geometry = comfy.memory_management.TensorGeometry(shape=weight.shape, dtype=model_dtype)- weight_size = geometry.numel() * geometry.element_size()- if vbar is not None and not hasattr(weight, "_v"):- weight._v = vbar.alloc(weight_size)- weight._model_dtype = model_dtype- allocated_size += weight_size- vbar.set_watermark_limit(allocated_size)+ if key not in self.backup:+ self.backup[key] = collections.namedtuple('Dimension', ['weight', 'inplace_update'])(weight, False)+ comfy.utils.set_attr_param(self.model, key, weight.to(device=device_to))+ self.model.model_loaded_weight_memory += weight.numel() * weight.element_size()move_weight_functions(m, device_to)- logging.info(f"Model {self.model.__class__.__name__} prepared for dynamic VRAM loading. {allocated_size // (1024 ** 2)}MB Staged. {num_patches} patches attached.")+ force_load_stat = f" Force pre-loaded {len(self.backup)} weights: {self.model.model_loaded_weight_memory // 1024} KB." if len(self.backup) > 0 else ""+ logging.info(f"Model {self.model.__class__.__name__} prepared for dynamic VRAM loading. {allocated_size // (1024 ** 2)}MB Staged. {num_patches} patches attached.{force_load_stat}")self.model.device = device_toself.model.current_weight_patches_uuid = self.patches_uuid@@ -1613,7 +1601,16 @@ class ModelPatcherDynamic(ModelPatcher):assert self.load_device != torch.device("cpu")vbar = self._vbar_get()- return 0 if vbar is None else vbar.free_memory(memory_to_free)+ freed = 0 if vbar is None else vbar.free_memory(memory_to_free)++ if freed < memory_to_free:+ for key in list(self.backup.keys()):+ bk = self.backup.pop(key)+ comfy.utils.set_attr_param(self.model, key, bk.weight)+ freed += self.model.model_loaded_weight_memory+ self.model.model_loaded_weight_memory = 0++ return freeddef partially_unload_ram(self, ram_to_unload):loading = self._load_list(prio_comfy_cast_weights=True, default_device=self.offload_device)@@ -1640,11 +1637,6 @@ class ModelPatcherDynamic(ModelPatcher):for m in self.model.modules():move_weight_functions(m, device_to)- keys = list(self.backup.keys())- for k in keys:- bk = self.backup[k]- comfy.utils.set_attr_param(self.model, k, bk.weight)-def partially_load(self, device_to, extra_memory=0, force_patch_weights=False):assert not force_patch_weights #See abovewith self.use_ejected(skip_and_inject_on_exit_only=True):