The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Move ModelPatcher to model_patcher.py
comfyanonymousf92074b84f44
diff --git a/comfy/controlnet.py b/comfy/controlnet.pyindex 4e20100c3..b18ab967a 100644--- a/comfy/controlnet.py+++ b/comfy/controlnet.py@@ -1,9 +1,9 @@import torchimport mathimport comfy.utils-import comfy.sdimport comfy.model_managementimport comfy.model_detection+import comfy.model_patcherimport comfy.cldm.cldmimport comfy.t2i_adapter.adapter@@ -129,7 +129,7 @@ class ControlNet(ControlBase):def __init__(self, control_model, global_average_pooling=False, device=None):super().__init__(device)self.control_model = control_model- self.control_model_wrapped = comfy.sd.ModelPatcher(self.control_model, load_device=comfy.model_management.get_torch_device(), offload_device=comfy.model_management.unet_offload_device())+ self.control_model_wrapped = comfy.model_patcher.ModelPatcher(self.control_model, load_device=comfy.model_management.get_torch_device(), offload_device=comfy.model_management.unet_offload_device())self.global_average_pooling = global_average_poolingdef get_control(self, x_noisy, t, cond, batched_number):diff --git a/comfy/model_patcher.py b/comfy/model_patcher.pynew file mode 100644index 000000000..2f087a600--- /dev/null+++ b/comfy/model_patcher.py@@ -0,0 +1,270 @@+import torch+import copy+import inspect++import comfy.utils++class ModelPatcher:+ def __init__(self, model, load_device, offload_device, size=0, current_device=None):+ self.size = size+ self.model = model+ self.patches = {}+ self.backup = {}+ self.model_options = {"transformer_options":{}}+ self.model_size()+ self.load_device = load_device+ self.offload_device = offload_device+ if current_device is None:+ self.current_device = self.offload_device+ else:+ self.current_device = current_device++ def model_size(self):+ if self.size > 0:+ return self.size+ model_sd = self.model.state_dict()+ size = 0+ for k in model_sd:+ t = model_sd[k]+ size += t.nelement() * t.element_size()+ self.size = size+ self.model_keys = set(model_sd.keys())+ return size++ def clone(self):+ n = ModelPatcher(self.model, self.load_device, self.offload_device, self.size, self.current_device)+ n.patches = {}+ for k in self.patches:+ n.patches[k] = self.patches[k][:]++ n.model_options = copy.deepcopy(self.model_options)+ n.model_keys = self.model_keys+ return n++ def is_clone(self, other):+ if hasattr(other, 'model') and self.model is other.model:+ return True+ return False++ def set_model_sampler_cfg_function(self, sampler_cfg_function):+ if len(inspect.signature(sampler_cfg_function).parameters) == 3:+ self.model_options["sampler_cfg_function"] = lambda args: sampler_cfg_function(args["cond"], args["uncond"], args["cond_scale"]) #Old way+ else:+ self.model_options["sampler_cfg_function"] = sampler_cfg_function++ def set_model_unet_function_wrapper(self, unet_wrapper_function):+ self.model_options["model_function_wrapper"] = unet_wrapper_function++ def set_model_patch(self, patch, name):+ to = self.model_options["transformer_options"]+ if "patches" not in to:+ to["patches"] = {}+ to["patches"][name] = to["patches"].get(name, []) + [patch]++ def set_model_patch_replace(self, patch, name, block_name, number):+ to = self.model_options["transformer_options"]+ if "patches_replace" not in to:+ to["patches_replace"] = {}+ if name not in to["patches_replace"]:+ to["patches_replace"][name] = {}+ to["patches_replace"][name][(block_name, number)] = patch++ def set_model_attn1_patch(self, patch):+ self.set_model_patch(patch, "attn1_patch")++ def set_model_attn2_patch(self, patch):+ self.set_model_patch(patch, "attn2_patch")++ def set_model_attn1_replace(self, patch, block_name, number):+ self.set_model_patch_replace(patch, "attn1", block_name, number)++ def set_model_attn2_replace(self, patch, block_name, number):+ self.set_model_patch_replace(patch, "attn2", block_name, number)++ def set_model_attn1_output_patch(self, patch):+ self.set_model_patch(patch, "attn1_output_patch")++ def set_model_attn2_output_patch(self, patch):+ self.set_model_patch(patch, "attn2_output_patch")++ def model_patches_to(self, device):+ to = self.model_options["transformer_options"]+ if "patches" in to:+ patches = to["patches"]+ for name in patches:+ patch_list = patches[name]+ for i in range(len(patch_list)):+ if hasattr(patch_list[i], "to"):+ patch_list[i] = patch_list[i].to(device)+ if "patches_replace" in to:+ patches = to["patches_replace"]+ for name in patches:+ patch_list = patches[name]+ for k in patch_list:+ if hasattr(patch_list[k], "to"):+ patch_list[k] = patch_list[k].to(device)++ def model_dtype(self):+ if hasattr(self.model, "get_dtype"):+ return self.model.get_dtype()++ def add_patches(self, patches, strength_patch=1.0, strength_model=1.0):+ p = set()+ for k in patches:+ if k in self.model_keys:+ p.add(k)+ current_patches = self.patches.get(k, [])+ current_patches.append((strength_patch, patches[k], strength_model))+ self.patches[k] = current_patches++ return list(p)++ def get_key_patches(self, filter_prefix=None):+ model_sd = self.model_state_dict()+ p = {}+ for k in model_sd:+ if filter_prefix is not None:+ if not k.startswith(filter_prefix):+ continue+ if k in self.patches:+ p[k] = [model_sd[k]] + self.patches[k]+ else:+ p[k] = (model_sd[k],)+ return p++ def model_state_dict(self, filter_prefix=None):+ sd = self.model.state_dict()+ keys = list(sd.keys())+ if filter_prefix is not None:+ for k in keys:+ if not k.startswith(filter_prefix):+ sd.pop(k)+ return sd++ def patch_model(self, device_to=None):+ model_sd = self.model_state_dict()+ for key in self.patches:+ if key not in model_sd:+ print("could not patch. key doesn't exist in model:", k)+ continue++ weight = model_sd[key]++ if key not in self.backup:+ self.backup[key] = weight.to(self.offload_device)++ if device_to is not None:+ temp_weight = weight.float().to(device_to, copy=True)+ else:+ temp_weight = weight.to(torch.float32, copy=True)+ out_weight = self.calculate_weight(self.patches[key], temp_weight, key).to(weight.dtype)+ comfy.utils.set_attr(self.model, key, out_weight)+ del temp_weight++ if device_to is not None:+ self.model.to(device_to)+ self.current_device = device_to++ return self.model++ def calculate_weight(self, patches, weight, key):+ for p in patches:+ alpha = p[0]+ v = p[1]+ strength_model = p[2]++ if strength_model != 1.0:+ weight *= strength_model++ if isinstance(v, list):+ v = (self.calculate_weight(v[1:], v[0].clone(), key), )++ if len(v) == 1:+ w1 = v[0]+ if alpha != 0.0:+ if w1.shape != weight.shape:+ print("WARNING SHAPE MISMATCH {} WEIGHT NOT MERGED {} != {}".format(key, w1.shape, weight.shape))+ else:+ weight += alpha * w1.type(weight.dtype).to(weight.device)+ elif len(v) == 4: #lora/locon+ mat1 = v[0].float().to(weight.device)+ mat2 = v[1].float().to(weight.device)+ if v[2] is not None:+ alpha *= v[2] / mat2.shape[0]+ if v[3] is not None:+ #locon mid weights, hopefully the math is fine because I didn't properly test it+ mat3 = v[3].float().to(weight.device)+ final_shape = [mat2.shape[1], mat2.shape[0], mat3.shape[2], mat3.shape[3]]+ mat2 = torch.mm(mat2.transpose(0, 1).flatten(start_dim=1), mat3.transpose(0, 1).flatten(start_dim=1)).reshape(final_shape).transpose(0, 1)+ try:+ weight += (alpha * torch.mm(mat1.flatten(start_dim=1), mat2.flatten(start_dim=1))).reshape(weight.shape).type(weight.dtype)+ except Exception as e:+ print("ERROR", key, e)+ elif len(v) == 8: #lokr+ w1 = v[0]+ w2 = v[1]+ w1_a = v[3]+ w1_b = v[4]+ w2_a = v[5]+ w2_b = v[6]+ t2 = v[7]+ dim = None++ if w1 is None:+ dim = w1_b.shape[0]+ w1 = torch.mm(w1_a.float(), w1_b.float())+ else:+ w1 = w1.float().to(weight.device)++ if w2 is None:+ dim = w2_b.shape[0]+ if t2 is None:+ w2 = torch.mm(w2_a.float().to(weight.device), w2_b.float().to(weight.device))+ else:+ w2 = torch.einsum('i j k l, j r, i p -> p r k l', t2.float().to(weight.device), w2_b.float().to(weight.device), w2_a.float().to(weight.device))+ else:+ w2 = w2.float().to(weight.device)++ if len(w2.shape) == 4:+ w1 = w1.unsqueeze(2).unsqueeze(2)+ if v[2] is not None and dim is not None:+ alpha *= v[2] / dim++ try:+ weight += alpha * torch.kron(w1, w2).reshape(weight.shape).type(weight.dtype)+ except Exception as e:+ print("ERROR", key, e)+ else: #loha+ w1a = v[0]+ w1b = v[1]+ if v[2] is not None:+ alpha *= v[2] / w1b.shape[0]+ w2a = v[3]+ w2b = v[4]+ if v[5] is not None: #cp decomposition+ t1 = v[5]+ t2 = v[6]+ m1 = torch.einsum('i j k l, j r, i p -> p r k l', t1.float().to(weight.device), w1b.float().to(weight.device), w1a.float().to(weight.device))+ m2 = torch.einsum('i j k l, j r, i p -> p r k l', t2.float().to(weight.device), w2b.float().to(weight.device), w2a.float().to(weight.device))+ else:+ m1 = torch.mm(w1a.float().to(weight.device), w1b.float().to(weight.device))+ m2 = torch.mm(w2a.float().to(weight.device), w2b.float().to(weight.device))++ try:+ weight += (alpha * m1 * m2).reshape(weight.shape).type(weight.dtype)+ except Exception as e:+ print("ERROR", key, e)++ return weight++ def unpatch_model(self, device_to=None):+ keys = list(self.backup.keys())++ for k in keys:+ comfy.utils.set_attr(self.model, k, self.backup[k])++ self.backup = {}++ if device_to is not None:+ self.model.to(device_to)+ self.current_device = device_todiff --git a/comfy/sd.py b/comfy/sd.pyindex 7462c79ef..1edc5fd29 100644--- a/comfy/sd.py+++ b/comfy/sd.py@@ -1,7 +1,5 @@import torchimport contextlib-import copy-import inspectimport mathfrom comfy import model_management@@ -21,6 +19,7 @@ from . import sd1_clipfrom . import sd2_clipfrom . import sdxl_clip+import comfy.model_patcherimport comfy.loraimport comfy.t2i_adapter.adapter@@ -53,271 +52,6 @@ def load_clip_weights(model, sd):sd = comfy.utils.transformers_convert(sd, "cond_stage_model.model.", "cond_stage_model.transformer.text_model.", 24)return load_model_weights(model, sd)-class ModelPatcher:- def __init__(self, model, load_device, offload_device, size=0, current_device=None):- self.size = size- self.model = model- self.patches = {}- self.backup = {}- self.model_options = {"transformer_options":{}}- self.model_size()- self.load_device = load_device- self.offload_device = offload_device- if current_device is None:- self.current_device = self.offload_device- else:- self.current_device = current_device-- def model_size(self):- if self.size > 0:- return self.size- model_sd = self.model.state_dict()- size = 0- for k in model_sd:- t = model_sd[k]- size += t.nelement() * t.element_size()- self.size = size- self.model_keys = set(model_sd.keys())- return size-- def clone(self):- n = ModelPatcher(self.model, self.load_device, self.offload_device, self.size, self.current_device)- n.patches = {}- for k in self.patches:- n.patches[k] = self.patches[k][:]-- n.model_options = copy.deepcopy(self.model_options)- n.model_keys = self.model_keys- return n-- def is_clone(self, other):- if hasattr(other, 'model') and self.model is other.model:- return True- return False-- def set_model_sampler_cfg_function(self, sampler_cfg_function):- if len(inspect.signature(sampler_cfg_function).parameters) == 3:- self.model_options["sampler_cfg_function"] = lambda args: sampler_cfg_function(args["cond"], args["uncond"], args["cond_scale"]) #Old way- else:- self.model_options["sampler_cfg_function"] = sampler_cfg_function-- def set_model_unet_function_wrapper(self, unet_wrapper_function):- self.model_options["model_function_wrapper"] = unet_wrapper_function-- def set_model_patch(self, patch, name):- to = self.model_options["transformer_options"]- if "patches" not in to:- to["patches"] = {}- to["patches"][name] = to["patches"].get(name, []) + [patch]-- def set_model_patch_replace(self, patch, name, block_name, number):- to = self.model_options["transformer_options"]- if "patches_replace" not in to:- to["patches_replace"] = {}- if name not in to["patches_replace"]:- to["patches_replace"][name] = {}- to["patches_replace"][name][(block_name, number)] = patch-- def set_model_attn1_patch(self, patch):- self.set_model_patch(patch, "attn1_patch")-- def set_model_attn2_patch(self, patch):- self.set_model_patch(patch, "attn2_patch")-- def set_model_attn1_replace(self, patch, block_name, number):- self.set_model_patch_replace(patch, "attn1", block_name, number)-- def set_model_attn2_replace(self, patch, block_name, number):- self.set_model_patch_replace(patch, "attn2", block_name, number)-- def set_model_attn1_output_patch(self, patch):- self.set_model_patch(patch, "attn1_output_patch")-- def set_model_attn2_output_patch(self, patch):- self.set_model_patch(patch, "attn2_output_patch")-- def model_patches_to(self, device):- to = self.model_options["transformer_options"]- if "patches" in to:- patches = to["patches"]- for name in patches:- patch_list = patches[name]- for i in range(len(patch_list)):- if hasattr(patch_list[i], "to"):- patch_list[i] = patch_list[i].to(device)- if "patches_replace" in to:- patches = to["patches_replace"]- for name in patches:- patch_list = patches[name]- for k in patch_list:- if hasattr(patch_list[k], "to"):- patch_list[k] = patch_list[k].to(device)-- def model_dtype(self):- if hasattr(self.model, "get_dtype"):- return self.model.get_dtype()-- def add_patches(self, patches, strength_patch=1.0, strength_model=1.0):- p = set()- for k in patches:- if k in self.model_keys:- p.add(k)- current_patches = self.patches.get(k, [])- current_patches.append((strength_patch, patches[k], strength_model))- self.patches[k] = current_patches-- return list(p)-- def get_key_patches(self, filter_prefix=None):- model_sd = self.model_state_dict()- p = {}- for k in model_sd:- if filter_prefix is not None:- if not k.startswith(filter_prefix):- continue- if k in self.patches:- p[k] = [model_sd[k]] + self.patches[k]- else:- p[k] = (model_sd[k],)- return p-- def model_state_dict(self, filter_prefix=None):- sd = self.model.state_dict()- keys = list(sd.keys())- if filter_prefix is not None:- for k in keys:- if not k.startswith(filter_prefix):- sd.pop(k)- return sd-- def patch_model(self, device_to=None):- model_sd = self.model_state_dict()- for key in self.patches:- if key not in model_sd:- print("could not patch. key doesn't exist in model:", k)- continue-- weight = model_sd[key]-- if key not in self.backup:- self.backup[key] = weight.to(self.offload_device)-- if device_to is not None:- temp_weight = weight.float().to(device_to, copy=True)- else:- temp_weight = weight.to(torch.float32, copy=True)- out_weight = self.calculate_weight(self.patches[key], temp_weight, key).to(weight.dtype)- comfy.utils.set_attr(self.model, key, out_weight)- del temp_weight-- if device_to is not None:- self.model.to(device_to)- self.current_device = device_to-- return self.model-- def calculate_weight(self, patches, weight, key):- for p in patches:- alpha = p[0]- v = p[1]- strength_model = p[2]-- if strength_model != 1.0:- weight *= strength_model-- if isinstance(v, list):- v = (self.calculate_weight(v[1:], v[0].clone(), key), )-- if len(v) == 1:- w1 = v[0]- if alpha != 0.0:- if w1.shape != weight.shape:- print("WARNING SHAPE MISMATCH {} WEIGHT NOT MERGED {} != {}".format(key, w1.shape, weight.shape))- else:- weight += alpha * w1.type(weight.dtype).to(weight.device)- elif len(v) == 4: #lora/locon- mat1 = v[0].float().to(weight.device)- mat2 = v[1].float().to(weight.device)- if v[2] is not None:- alpha *= v[2] / mat2.shape[0]- if v[3] is not None:- #locon mid weights, hopefully the math is fine because I didn't properly test it- mat3 = v[3].float().to(weight.device)- final_shape = [mat2.shape[1], mat2.shape[0], mat3.shape[2], mat3.shape[3]]- mat2 = torch.mm(mat2.transpose(0, 1).flatten(start_dim=1), mat3.transpose(0, 1).flatten(start_dim=1)).reshape(final_shape).transpose(0, 1)- try:- weight += (alpha * torch.mm(mat1.flatten(start_dim=1), mat2.flatten(start_dim=1))).reshape(weight.shape).type(weight.dtype)- except Exception as e:- print("ERROR", key, e)- elif len(v) == 8: #lokr- w1 = v[0]- w2 = v[1]- w1_a = v[3]- w1_b = v[4]- w2_a = v[5]- w2_b = v[6]- t2 = v[7]- dim = None-- if w1 is None:- dim = w1_b.shape[0]- w1 = torch.mm(w1_a.float(), w1_b.float())- else:- w1 = w1.float().to(weight.device)-- if w2 is None:- dim = w2_b.shape[0]- if t2 is None:- w2 = torch.mm(w2_a.float().to(weight.device), w2_b.float().to(weight.device))- else:- w2 = torch.einsum('i j k l, j r, i p -> p r k l', t2.float().to(weight.device), w2_b.float().to(weight.device), w2_a.float().to(weight.device))- else:- w2 = w2.float().to(weight.device)-- if len(w2.shape) == 4:- w1 = w1.unsqueeze(2).unsqueeze(2)- if v[2] is not None and dim is not None:- alpha *= v[2] / dim-- try:- weight += alpha * torch.kron(w1, w2).reshape(weight.shape).type(weight.dtype)- except Exception as e:- print("ERROR", key, e)- else: #loha- w1a = v[0]- w1b = v[1]- if v[2] is not None:- alpha *= v[2] / w1b.shape[0]- w2a = v[3]- w2b = v[4]- if v[5] is not None: #cp decomposition- t1 = v[5]- t2 = v[6]- m1 = torch.einsum('i j k l, j r, i p -> p r k l', t1.float().to(weight.device), w1b.float().to(weight.device), w1a.float().to(weight.device))- m2 = torch.einsum('i j k l, j r, i p -> p r k l', t2.float().to(weight.device), w2b.float().to(weight.device), w2a.float().to(weight.device))- else:- m1 = torch.mm(w1a.float().to(weight.device), w1b.float().to(weight.device))- m2 = torch.mm(w2a.float().to(weight.device), w2b.float().to(weight.device))-- try:- weight += (alpha * m1 * m2).reshape(weight.shape).type(weight.dtype)- except Exception as e:- print("ERROR", key, e)-- return weight-- def unpatch_model(self, device_to=None):- keys = list(self.backup.keys())-- for k in keys:- comfy.utils.set_attr(self.model, k, self.backup[k])-- self.backup = {}-- if device_to is not None:- self.model.to(device_to)- self.current_device = device_to-def load_lora_for_models(model, clip, lora, strength_model, strength_clip):key_map = comfy.lora.model_lora_keys_unet(model.model)@@ -355,7 +89,7 @@ class CLIP:self.cond_stage_model = clip(**(params))self.tokenizer = tokenizer(embedding_directory=embedding_directory)- self.patcher = ModelPatcher(self.cond_stage_model, load_device=load_device, offload_device=offload_device)+ self.patcher = comfy.model_patcher.ModelPatcher(self.cond_stage_model, load_device=load_device, offload_device=offload_device)self.layer_idx = Nonedef clone(self):@@ -573,7 +307,7 @@ def load_gligen(ckpt_path):model = gligen.load_gligen(data)if model_management.should_use_fp16():model = model.half()- return ModelPatcher(model, load_device=model_management.get_torch_device(), offload_device=model_management.unet_offload_device())+ return comfy.model_patcher.ModelPatcher(model, load_device=model_management.get_torch_device(), offload_device=model_management.unet_offload_device())def load_checkpoint(config_path=None, ckpt_path=None, output_vae=True, output_clip=True, embedding_directory=None, state_dict=None, config=None):#TODO: this function is a mess and should be removed eventually@@ -653,7 +387,7 @@ def load_checkpoint(config_path=None, ckpt_path=None, output_vae=True, output_clw.cond_stage_model = clip.cond_stage_modelload_clip_weights(w, state_dict)- return (ModelPatcher(model, load_device=model_management.get_torch_device(), offload_device=offload_device), clip, vae)+ return (comfy.model_patcher.ModelPatcher(model, load_device=model_management.get_torch_device(), offload_device=offload_device), clip, vae)def load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=True, output_clipvision=False, embedding_directory=None):sd = comfy.utils.load_torch_file(ckpt_path)@@ -705,7 +439,7 @@ def load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=True, oif len(left_over) > 0:print("left over keys:", left_over)- model_patcher = ModelPatcher(model, load_device=model_management.get_torch_device(), offload_device=model_management.unet_offload_device(), current_device=inital_load_device)+ model_patcher = comfy.model_patcher.ModelPatcher(model, load_device=model_management.get_torch_device(), offload_device=model_management.unet_offload_device(), current_device=inital_load_device)if inital_load_device != torch.device("cpu"):print("loaded straight to GPU")model_management.load_model_gpu(model_patcher)@@ -735,7 +469,7 @@ def load_unet(unet_path): #load unet in diffusers formatmodel = model_config.get_model(new_sd, "")model = model.to(offload_device)model.load_model_weights(new_sd, "")- return ModelPatcher(model, load_device=model_management.get_torch_device(), offload_device=offload_device)+ return comfy.model_patcher.ModelPatcher(model, load_device=model_management.get_torch_device(), offload_device=offload_device)def save_checkpoint(output_path, model, clip, vae, metadata=None):model_management.load_models_gpu([model, clip.load_model()])