The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Lower memory usage for loras in lowvram mode at the cost of perf.
comfyanonymousdb8b59ecff7b
diff --git a/comfy/model_management.py b/comfy/model_management.pyindex 2f0a0a627..66fa918bb 100644--- a/comfy/model_management.py+++ b/comfy/model_management.py@@ -272,7 +272,6 @@ def module_size(module):class LoadedModel:def __init__(self, model):self.model = model- self.model_accelerated = Falseself.device = model.load_devicedef model_memory(self):@@ -285,52 +284,27 @@ class LoadedModel:return self.model_memory()def model_load(self, lowvram_model_memory=0):- patch_model_to = None- if lowvram_model_memory == 0:- patch_model_to = self.device+ patch_model_to = self.deviceself.model.model_patches_to(self.device)self.model.model_patches_to(self.model.model_dtype())try:- self.real_model = self.model.patch_model(device_to=patch_model_to) #TODO: do something with loras and offloading to CPU+ if lowvram_model_memory > 0:+ self.real_model = self.model.patch_model_lowvram(device_to=patch_model_to, lowvram_model_memory=lowvram_model_memory)+ else:+ self.real_model = self.model.patch_model(device_to=patch_model_to)except Exception as e:self.model.unpatch_model(self.model.offload_device)self.model_unload()raise e- if lowvram_model_memory > 0:- logging.info("loading in lowvram mode {}".format(lowvram_model_memory/(1024 * 1024)))- mem_counter = 0- for m in self.real_model.modules():- if hasattr(m, "comfy_cast_weights"):- m.prev_comfy_cast_weights = m.comfy_cast_weights- m.comfy_cast_weights = True- module_mem = module_size(m)- if mem_counter + module_mem < lowvram_model_memory:- m.to(self.device)- mem_counter += module_mem- elif hasattr(m, "weight"): #only modules with comfy_cast_weights can be set to lowvram mode- m.to(self.device)- mem_counter += module_size(m)- logging.warning("lowvram: loaded module regularly {}".format(m))-- self.model_accelerated = True-if is_intel_xpu() and not args.disable_ipex_optimize:self.real_model = torch.xpu.optimize(self.real_model.eval(), inplace=True, auto_kernel_selection=True, graph_mode=True)return self.real_modeldef model_unload(self):- if self.model_accelerated:- for m in self.real_model.modules():- if hasattr(m, "prev_comfy_cast_weights"):- m.comfy_cast_weights = m.prev_comfy_cast_weights- del m.prev_comfy_cast_weights-- self.model_accelerated = False-self.model.unpatch_model(self.model.offload_device)self.model.model_patches_to(self.model.offload_device)diff --git a/comfy/model_patcher.py b/comfy/model_patcher.pyindex 5e578dffc..475fa812c 100644--- a/comfy/model_patcher.py+++ b/comfy/model_patcher.py@@ -24,6 +24,7 @@ class ModelPatcher:self.current_device = current_deviceself.weight_inplace_update = weight_inplace_update+ self.model_lowvram = Falsedef model_size(self):if self.size > 0:@@ -178,6 +179,27 @@ class ModelPatcher:sd.pop(k)return sd+ def patch_weight_to_device(self, key, device_to=None):+ if key not in self.patches:+ return++ weight = comfy.utils.get_attr(self.model, key)++ inplace_update = self.weight_inplace_update++ if key not in self.backup:+ self.backup[key] = weight.to(device=self.offload_device, copy=inplace_update)++ if device_to is not None:+ temp_weight = comfy.model_management.cast_to_device(weight, device_to, torch.float32, copy=True)+ else:+ temp_weight = weight.to(torch.float32, copy=True)+ out_weight = self.calculate_weight(self.patches[key], temp_weight, key).to(weight.dtype)+ if inplace_update:+ comfy.utils.copy_to_param(self.model, key, out_weight)+ else:+ comfy.utils.set_attr_param(self.model, key, out_weight)+def patch_model(self, device_to=None, patch_weights=True):for k in self.object_patches:old = comfy.utils.set_attr(self.model, k, self.object_patches[k])@@ -191,23 +213,7 @@ class ModelPatcher:logging.warning("could not patch. key doesn't exist in model: {}".format(key))continue- weight = model_sd[key]-- inplace_update = self.weight_inplace_update-- if key not in self.backup:- self.backup[key] = weight.to(device=self.offload_device, copy=inplace_update)-- if device_to is not None:- temp_weight = comfy.model_management.cast_to_device(weight, device_to, torch.float32, copy=True)- else:- temp_weight = weight.to(torch.float32, copy=True)- out_weight = self.calculate_weight(self.patches[key], temp_weight, key).to(weight.dtype)- if inplace_update:- comfy.utils.copy_to_param(self.model, key, out_weight)- else:- comfy.utils.set_attr_param(self.model, key, out_weight)- del temp_weight+ self.patch_weight_to_device(key, device_to)if device_to is not None:self.model.to(device_to)@@ -215,6 +221,47 @@ class ModelPatcher:return self.model+ def patch_model_lowvram(self, device_to=None, lowvram_model_memory=0):+ self.patch_model(device_to, patch_weights=False)++ logging.info("loading in lowvram mode {}".format(lowvram_model_memory/(1024 * 1024)))+ class LowVramPatch:+ def __init__(self, key, model_patcher):+ self.key = key+ self.model_patcher = model_patcher+ def __call__(self, weight):+ return self.model_patcher.calculate_weight(self.model_patcher.patches[self.key], weight, self.key)++ mem_counter = 0+ for n, m in self.model.named_modules():+ lowvram_weight = False+ if hasattr(m, "comfy_cast_weights"):+ module_mem = comfy.model_management.module_size(m)+ if mem_counter + module_mem >= lowvram_model_memory:+ lowvram_weight = True++ weight_key = "{}.weight".format(n)+ bias_key = "{}.bias".format(n)++ if lowvram_weight:+ if weight_key in self.patches:+ m.weight_function = LowVramPatch(weight_key, self)+ if bias_key in self.patches:+ m.bias_function = LowVramPatch(weight_key, self)++ m.prev_comfy_cast_weights = m.comfy_cast_weights+ m.comfy_cast_weights = True+ else:+ if hasattr(m, "weight"):+ self.patch_weight_to_device(weight_key, device_to)+ self.patch_weight_to_device(bias_key, device_to)+ m.to(device_to)+ mem_counter += comfy.model_management.module_size(m)+ logging.debug("lowvram: loaded module regularly {}".format(m))++ self.model_lowvram = True+ return self.model+def calculate_weight(self, patches, weight, key):for p in patches:alpha = p[0]@@ -341,6 +388,16 @@ class ModelPatcher:return weightdef unpatch_model(self, device_to=None):+ if self.model_lowvram:+ for m in self.model.modules():+ if hasattr(m, "prev_comfy_cast_weights"):+ m.comfy_cast_weights = m.prev_comfy_cast_weights+ del m.prev_comfy_cast_weights+ m.weight_function = None+ m.bias_function = None++ self.model_lowvram = False+keys = list(self.backup.keys())if self.weight_inplace_update:diff --git a/comfy/ops.py b/comfy/ops.pyindex 517688e8b..cfdec355c 100644--- a/comfy/ops.py+++ b/comfy/ops.py@@ -24,13 +24,20 @@ def cast_bias_weight(s, input):non_blocking = comfy.model_management.device_supports_non_blocking(input.device)if s.bias is not None:bias = s.bias.to(device=input.device, dtype=input.dtype, non_blocking=non_blocking)+ if s.bias_function is not None:+ bias = s.bias_function(bias)weight = s.weight.to(device=input.device, dtype=input.dtype, non_blocking=non_blocking)+ if s.weight_function is not None:+ weight = s.weight_function(weight)return weight, biasclass disable_weight_init:class Linear(torch.nn.Linear):comfy_cast_weights = False+ weight_function = None+ bias_function = None+def reset_parameters(self):return None@@ -46,6 +53,9 @@ class disable_weight_init:class Conv2d(torch.nn.Conv2d):comfy_cast_weights = False+ weight_function = None+ bias_function = None+def reset_parameters(self):return None@@ -61,6 +71,9 @@ class disable_weight_init:class Conv3d(torch.nn.Conv3d):comfy_cast_weights = False+ weight_function = None+ bias_function = None+def reset_parameters(self):return None@@ -76,6 +89,9 @@ class disable_weight_init:class GroupNorm(torch.nn.GroupNorm):comfy_cast_weights = False+ weight_function = None+ bias_function = None+def reset_parameters(self):return None@@ -92,6 +108,9 @@ class disable_weight_init:class LayerNorm(torch.nn.LayerNorm):comfy_cast_weights = False+ weight_function = None+ bias_function = None+def reset_parameters(self):return None@@ -111,6 +130,9 @@ class disable_weight_init:class ConvTranspose2d(torch.nn.ConvTranspose2d):comfy_cast_weights = False+ weight_function = None+ bias_function = None+def reset_parameters(self):return None