The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Support controlnet in diffusers format.
comfyanonymous78e7958d17fd
diff --git a/comfy/model_detection.py b/comfy/model_detection.pyindex cf764e0b7..691d4c6c4 100644--- a/comfy/model_detection.py+++ b/comfy/model_detection.py@@ -118,3 +118,57 @@ def model_config_from_unet_config(unet_config):def model_config_from_unet(state_dict, unet_key_prefix, use_fp16):unet_config = detect_unet_config(state_dict, unet_key_prefix, use_fp16)return model_config_from_unet_config(unet_config)+++def model_config_from_diffusers_unet(state_dict, use_fp16):+ match = {}+ match["context_dim"] = state_dict["down_blocks.1.attentions.1.transformer_blocks.0.attn2.to_k.weight"].shape[1]+ match["model_channels"] = state_dict["conv_in.weight"].shape[0]+ match["in_channels"] = state_dict["conv_in.weight"].shape[1]+ match["adm_in_channels"] = None+ if "class_embedding.linear_1.weight" in state_dict:+ match["adm_in_channels"] = state_dict["class_embedding.linear_1.weight"].shape[1]+ elif "add_embedding.linear_1.weight" in state_dict:+ match["adm_in_channels"] = state_dict["add_embedding.linear_1.weight"].shape[1]++ SDXL = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,+ 'num_classes': 'sequential', 'adm_in_channels': 2816, 'use_fp16': use_fp16, 'in_channels': 4, 'model_channels': 320,+ 'num_res_blocks': 2, 'attention_resolutions': [2, 4], 'transformer_depth': [0, 2, 10], 'channel_mult': [1, 2, 4],+ 'transformer_depth_middle': 10, 'use_linear_in_transformer': True, 'context_dim': 2048}++ SDXL_refiner = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,+ 'num_classes': 'sequential', 'adm_in_channels': 2560, 'use_fp16': use_fp16, 'in_channels': 4, 'model_channels': 384,+ 'num_res_blocks': 2, 'attention_resolutions': [2, 4], 'transformer_depth': [0, 4, 4, 0], 'channel_mult': [1, 2, 4, 4],+ 'transformer_depth_middle': 4, 'use_linear_in_transformer': True, 'context_dim': 1280}++ SD21 = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,+ 'adm_in_channels': None, 'use_fp16': use_fp16, 'in_channels': 4, 'model_channels': 320, 'num_res_blocks': 2,+ 'attention_resolutions': [1, 2, 4], 'transformer_depth': [1, 1, 1, 0], 'channel_mult': [1, 2, 4, 4],+ 'transformer_depth_middle': 1, 'use_linear_in_transformer': True, 'context_dim': 1024}++ SD21_uncliph = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,+ 'num_classes': 'sequential', 'adm_in_channels': 2048, 'use_fp16': use_fp16, 'in_channels': 4, 'model_channels': 320,+ 'num_res_blocks': 2, 'attention_resolutions': [1, 2, 4], 'transformer_depth': [1, 1, 1, 0], 'channel_mult': [1, 2, 4, 4],+ 'transformer_depth_middle': 1, 'use_linear_in_transformer': True, 'context_dim': 1024}++ SD21_unclipl = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,+ 'num_classes': 'sequential', 'adm_in_channels': 1536, 'use_fp16': use_fp16, 'in_channels': 4, 'model_channels': 320,+ 'num_res_blocks': 2, 'attention_resolutions': [1, 2, 4], 'transformer_depth': [1, 1, 1, 0], 'channel_mult': [1, 2, 4, 4],+ 'transformer_depth_middle': 1, 'use_linear_in_transformer': True, 'context_dim': 1024}++ SD15 = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,+ 'adm_in_channels': None, 'use_fp16': use_fp16, 'in_channels': 4, 'model_channels': 320, 'num_res_blocks': 2,+ 'attention_resolutions': [1, 2, 4], 'transformer_depth': [1, 1, 1, 0], 'channel_mult': [1, 2, 4, 4],+ 'transformer_depth_middle': 1, 'use_linear_in_transformer': False, 'context_dim': 768}++ supported_models = [SDXL, SDXL_refiner, SD21, SD15, SD21_uncliph, SD21_unclipl]++ for unet_config in supported_models:+ matches = True+ for k in match:+ if match[k] != unet_config[k]:+ matches = False+ break+ if matches:+ return model_config_from_unet_config(unet_config)+ return Nonediff --git a/comfy/sd.py b/comfy/sd.pyindex 61b593837..7f7c06bc5 100644--- a/comfy/sd.py+++ b/comfy/sd.py@@ -765,6 +765,51 @@ class ControlNet:def load_controlnet(ckpt_path, model=None):controlnet_data = utils.load_torch_file(ckpt_path, safe_load=True)++ controlnet_config = None+ if "controlnet_cond_embedding.conv_in.weight" in controlnet_data: #diffusers format+ use_fp16 = model_management.should_use_fp16()+ controlnet_config = model_detection.model_config_from_diffusers_unet(controlnet_data, use_fp16).unet_config+ diffusers_keys = utils.unet_to_diffusers(controlnet_config)+ diffusers_keys["controlnet_mid_block.weight"] = "middle_block_out.0.weight"+ diffusers_keys["controlnet_mid_block.bias"] = "middle_block_out.0.bias"++ count = 0+ loop = True+ while loop:+ suffix = [".weight", ".bias"]+ for s in suffix:+ k_in = "controlnet_down_blocks.{}{}".format(count, s)+ k_out = "zero_convs.{}.0{}".format(count, s)+ if k_in not in controlnet_data:+ loop = False+ break+ diffusers_keys[k_in] = k_out+ count += 1++ count = 0+ loop = True+ while loop:+ suffix = [".weight", ".bias"]+ for s in suffix:+ if count == 0:+ k_in = "controlnet_cond_embedding.conv_in{}".format(s)+ else:+ k_in = "controlnet_cond_embedding.blocks.{}{}".format(count - 1, s)+ k_out = "input_hint_block.{}{}".format(count * 2, s)+ if k_in not in controlnet_data:+ k_in = "controlnet_cond_embedding.conv_out{}".format(s)+ loop = False+ diffusers_keys[k_in] = k_out+ count += 1++ new_sd = {}+ for k in diffusers_keys:+ if k in controlnet_data:+ new_sd[diffusers_keys[k]] = controlnet_data.pop(k)++ controlnet_data = new_sd+pth_key = 'control_model.zero_convs.0.0.weight'pth = Falsekey = 'zero_convs.0.0.weight'@@ -780,9 +825,9 @@ def load_controlnet(ckpt_path, model=None):print("error checkpoint does not contain controlnet or t2i adapter data", ckpt_path)return net- use_fp16 = model_management.should_use_fp16()-- controlnet_config = model_detection.model_config_from_unet(controlnet_data, prefix, use_fp16).unet_config+ if controlnet_config is None:+ use_fp16 = model_management.should_use_fp16()+ controlnet_config = model_detection.model_config_from_unet(controlnet_data, prefix, use_fp16).unet_configcontrolnet_config.pop("out_channels")controlnet_config["hint_channels"] = 3control_model = cldm.ControlNet(**controlnet_config)@@ -1140,69 +1185,24 @@ def load_unet(unet_path): #load unet in diffusers formatparameters = calculate_parameters(sd, "")fp16 = model_management.should_use_fp16(model_params=parameters)- match = {}- match["context_dim"] = sd["down_blocks.1.attentions.1.transformer_blocks.0.attn2.to_k.weight"].shape[1]- match["model_channels"] = sd["conv_in.weight"].shape[0]- match["in_channels"] = sd["conv_in.weight"].shape[1]- match["adm_in_channels"] = None- if "class_embedding.linear_1.weight" in sd:- match["adm_in_channels"] = sd["class_embedding.linear_1.weight"].shape[1]- elif "add_embedding.linear_1.weight" in sd:- match["adm_in_channels"] = sd["add_embedding.linear_1.weight"].shape[1]-- SDXL = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,- 'num_classes': 'sequential', 'adm_in_channels': 2816, 'use_fp16': fp16, 'in_channels': 4, 'model_channels': 320,- 'num_res_blocks': 2, 'attention_resolutions': [2, 4], 'transformer_depth': [0, 2, 10], 'channel_mult': [1, 2, 4],- 'transformer_depth_middle': 10, 'use_linear_in_transformer': True, 'context_dim': 2048}-- SDXL_refiner = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,- 'num_classes': 'sequential', 'adm_in_channels': 2560, 'use_fp16': fp16, 'in_channels': 4, 'model_channels': 384,- 'num_res_blocks': 2, 'attention_resolutions': [2, 4], 'transformer_depth': [0, 4, 4, 0], 'channel_mult': [1, 2, 4, 4],- 'transformer_depth_middle': 4, 'use_linear_in_transformer': True, 'context_dim': 1280}-- SD21 = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,- 'adm_in_channels': None, 'use_fp16': fp16, 'in_channels': 4, 'model_channels': 320, 'num_res_blocks': 2,- 'attention_resolutions': [1, 2, 4], 'transformer_depth': [1, 1, 1, 0], 'channel_mult': [1, 2, 4, 4],- 'transformer_depth_middle': 1, 'use_linear_in_transformer': True, 'context_dim': 1024}-- SD21_uncliph = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,- 'num_classes': 'sequential', 'adm_in_channels': 2048, 'use_fp16': True, 'in_channels': 4, 'model_channels': 320,- 'num_res_blocks': 2, 'attention_resolutions': [1, 2, 4], 'transformer_depth': [1, 1, 1, 0], 'channel_mult': [1, 2, 4, 4],- 'transformer_depth_middle': 1, 'use_linear_in_transformer': True, 'context_dim': 1024}-- SD21_unclipl = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,- 'num_classes': 'sequential', 'adm_in_channels': 1536, 'use_fp16': True, 'in_channels': 4, 'model_channels': 320,- 'num_res_blocks': 2, 'attention_resolutions': [1, 2, 4], 'transformer_depth': [1, 1, 1, 0], 'channel_mult': [1, 2, 4, 4],- 'transformer_depth_middle': 1, 'use_linear_in_transformer': True, 'context_dim': 1024}-- SD15 = {'use_checkpoint': False, 'image_size': 32, 'out_channels': 4, 'use_spatial_transformer': True, 'legacy': False,- 'adm_in_channels': None, 'use_fp16': True, 'in_channels': 4, 'model_channels': 320, 'num_res_blocks': 2,- 'attention_resolutions': [1, 2, 4], 'transformer_depth': [1, 1, 1, 0], 'channel_mult': [1, 2, 4, 4],- 'transformer_depth_middle': 1, 'use_linear_in_transformer': False, 'context_dim': 768}-- supported_models = [SDXL, SDXL_refiner, SD21, SD15, SD21_uncliph, SD21_unclipl]- print("match", match)- for unet_config in supported_models:- matches = True- for k in match:- if match[k] != unet_config[k]:- matches = False- break- if matches:- diffusers_keys = utils.unet_to_diffusers(unet_config)- new_sd = {}- for k in diffusers_keys:- if k in sd:- new_sd[diffusers_keys[k]] = sd.pop(k)- else:- print(diffusers_keys[k], k)- offload_device = model_management.unet_offload_device()- model_config = model_detection.model_config_from_unet_config(unet_config)- model = model_config.get_model(new_sd, "")- model = model.to(offload_device)- model.load_model_weights(new_sd, "")- return ModelPatcher(model, load_device=model_management.get_torch_device(), offload_device=offload_device)- print("ERROR UNSUPPORTED UNET", unet_path)+ model_config = model_detection.model_config_from_diffusers_unet(sd, fp16)+ if model_config is None:+ print("ERROR UNSUPPORTED UNET", unet_path)+ return None++ diffusers_keys = utils.unet_to_diffusers(model_config.unet_config)++ new_sd = {}+ for k in diffusers_keys:+ if k in sd:+ new_sd[diffusers_keys[k]] = sd.pop(k)+ else:+ print(diffusers_keys[k], k)+ offload_device = model_management.unet_offload_device()+ model = model_config.get_model(new_sd, "")+ model = model.to(offload_device)+ model.load_model_weights(new_sd, "")+ return ModelPatcher(model, load_device=model_management.get_torch_device(), offload_device=offload_device)def save_checkpoint(output_path, model, clip, vae, metadata=None):try: