The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Add experimental photomaker nodes.
comfyanonymousd1533d9c0f1d
diff --git a/comfy_extras/nodes_photomaker.py b/comfy_extras/nodes_photomaker.pynew file mode 100644index 000000000..90130142b--- /dev/null+++ b/comfy_extras/nodes_photomaker.py@@ -0,0 +1,187 @@+import torch+import torch.nn as nn+import folder_paths+import comfy.clip_model+import comfy.clip_vision+import comfy.ops++# code for model from: https://github.com/TencentARC/PhotoMaker/blob/main/photomaker/model.py under Apache License Version 2.0+VISION_CONFIG_DICT = {+ "hidden_size": 1024,+ "image_size": 224,+ "intermediate_size": 4096,+ "num_attention_heads": 16,+ "num_channels": 3,+ "num_hidden_layers": 24,+ "patch_size": 14,+ "projection_dim": 768,+ "hidden_act": "quick_gelu",+}++class MLP(nn.Module):+ def __init__(self, in_dim, out_dim, hidden_dim, use_residual=True, operations=comfy.ops):+ super().__init__()+ if use_residual:+ assert in_dim == out_dim+ self.layernorm = operations.LayerNorm(in_dim)+ self.fc1 = operations.Linear(in_dim, hidden_dim)+ self.fc2 = operations.Linear(hidden_dim, out_dim)+ self.use_residual = use_residual+ self.act_fn = nn.GELU()++ def forward(self, x):+ residual = x+ x = self.layernorm(x)+ x = self.fc1(x)+ x = self.act_fn(x)+ x = self.fc2(x)+ if self.use_residual:+ x = x + residual+ return x+++class FuseModule(nn.Module):+ def __init__(self, embed_dim, operations):+ super().__init__()+ self.mlp1 = MLP(embed_dim * 2, embed_dim, embed_dim, use_residual=False, operations=operations)+ self.mlp2 = MLP(embed_dim, embed_dim, embed_dim, use_residual=True, operations=operations)+ self.layer_norm = operations.LayerNorm(embed_dim)++ def fuse_fn(self, prompt_embeds, id_embeds):+ stacked_id_embeds = torch.cat([prompt_embeds, id_embeds], dim=-1)+ stacked_id_embeds = self.mlp1(stacked_id_embeds) + prompt_embeds+ stacked_id_embeds = self.mlp2(stacked_id_embeds)+ stacked_id_embeds = self.layer_norm(stacked_id_embeds)+ return stacked_id_embeds++ def forward(+ self,+ prompt_embeds,+ id_embeds,+ class_tokens_mask,+ ) -> torch.Tensor:+ # id_embeds shape: [b, max_num_inputs, 1, 2048]+ id_embeds = id_embeds.to(prompt_embeds.dtype)+ num_inputs = class_tokens_mask.sum().unsqueeze(0) # TODO: check for training case+ batch_size, max_num_inputs = id_embeds.shape[:2]+ # seq_length: 77+ seq_length = prompt_embeds.shape[1]+ # flat_id_embeds shape: [b*max_num_inputs, 1, 2048]+ flat_id_embeds = id_embeds.view(+ -1, id_embeds.shape[-2], id_embeds.shape[-1]+ )+ # valid_id_mask [b*max_num_inputs]+ valid_id_mask = (+ torch.arange(max_num_inputs, device=flat_id_embeds.device)[None, :]+ < num_inputs[:, None]+ )+ valid_id_embeds = flat_id_embeds[valid_id_mask.flatten()]++ prompt_embeds = prompt_embeds.view(-1, prompt_embeds.shape[-1])+ class_tokens_mask = class_tokens_mask.view(-1)+ valid_id_embeds = valid_id_embeds.view(-1, valid_id_embeds.shape[-1])+ # slice out the image token embeddings+ image_token_embeds = prompt_embeds[class_tokens_mask]+ stacked_id_embeds = self.fuse_fn(image_token_embeds, valid_id_embeds)+ assert class_tokens_mask.sum() == stacked_id_embeds.shape[0], f"{class_tokens_mask.sum()} != {stacked_id_embeds.shape[0]}"+ prompt_embeds.masked_scatter_(class_tokens_mask[:, None], stacked_id_embeds.to(prompt_embeds.dtype))+ updated_prompt_embeds = prompt_embeds.view(batch_size, seq_length, -1)+ return updated_prompt_embeds++class PhotoMakerIDEncoder(comfy.clip_model.CLIPVisionModelProjection):+ def __init__(self):+ self.load_device = comfy.model_management.text_encoder_device()+ offload_device = comfy.model_management.text_encoder_offload_device()+ dtype = comfy.model_management.text_encoder_dtype(self.load_device)++ super().__init__(VISION_CONFIG_DICT, dtype, offload_device, comfy.ops.manual_cast)+ self.visual_projection_2 = comfy.ops.manual_cast.Linear(1024, 1280, bias=False)+ self.fuse_module = FuseModule(2048, comfy.ops.manual_cast)++ def forward(self, id_pixel_values, prompt_embeds, class_tokens_mask):+ b, num_inputs, c, h, w = id_pixel_values.shape+ id_pixel_values = id_pixel_values.view(b * num_inputs, c, h, w)++ shared_id_embeds = self.vision_model(id_pixel_values)[2]+ id_embeds = self.visual_projection(shared_id_embeds)+ id_embeds_2 = self.visual_projection_2(shared_id_embeds)++ id_embeds = id_embeds.view(b, num_inputs, 1, -1)+ id_embeds_2 = id_embeds_2.view(b, num_inputs, 1, -1)++ id_embeds = torch.cat((id_embeds, id_embeds_2), dim=-1)+ updated_prompt_embeds = self.fuse_module(prompt_embeds, id_embeds, class_tokens_mask)++ return updated_prompt_embeds+++class PhotoMakerLoader:+ @classmethod+ def INPUT_TYPES(s):+ return {"required": { "photomaker_model_name": (folder_paths.get_filename_list("photomaker"), )}}++ RETURN_TYPES = ("PHOTOMAKER",)+ FUNCTION = "load_photomaker_model"++ CATEGORY = "_for_testing/photomaker"++ def load_photomaker_model(self, photomaker_model_name):+ photomaker_model_path = folder_paths.get_full_path("photomaker", photomaker_model_name)+ photomaker_model = PhotoMakerIDEncoder()+ data = comfy.utils.load_torch_file(photomaker_model_path, safe_load=True)+ if "id_encoder" in data:+ data = data["id_encoder"]+ photomaker_model.load_state_dict(data)+ return (photomaker_model,)+++class PhotoMakerEncode:+ @classmethod+ def INPUT_TYPES(s):+ return {"required": { "photomaker": ("PHOTOMAKER",),+ "image": ("IMAGE",),+ "clip": ("CLIP", ),+ "text": ("STRING", {"multiline": True, "default": "photograph of photomaker"}),+ }}++ RETURN_TYPES = ("CONDITIONING",)+ FUNCTION = "apply_photomaker"++ CATEGORY = "_for_testing/photomaker"++ def apply_photomaker(self, photomaker, image, clip, text):+ special_token = "photomaker"+ pixel_values = comfy.clip_vision.clip_preprocess(image.to(photomaker.load_device)).float()+ try:+ index = text.split(" ").index(special_token) + 1+ except ValueError:+ index = -1+ tokens = clip.tokenize(text, return_word_ids=True)+ out_tokens = {}+ for k in tokens:+ out_tokens[k] = []+ for t in tokens[k]:+ f = list(filter(lambda x: x[2] != index, t))+ while len(f) < len(t):+ f.append(t[-1])+ out_tokens[k].append(f)++ cond, pooled = clip.encode_from_tokens(out_tokens, return_pooled=True)++ if index > 0:+ token_index = index - 1+ num_id_images = 1+ class_tokens_mask = [True if token_index <= i < token_index+num_id_images else False for i in range(77)]+ out = photomaker(id_pixel_values=pixel_values.unsqueeze(0), prompt_embeds=cond.to(photomaker.load_device),+ class_tokens_mask=torch.tensor(class_tokens_mask, dtype=torch.bool, device=photomaker.load_device).unsqueeze(0))+ else:+ out = cond++ return ([[out, {"pooled_output": pooled}]], )+++NODE_CLASS_MAPPINGS = {+ "PhotoMakerLoader": PhotoMakerLoader,+ "PhotoMakerEncode": PhotoMakerEncode,+}+diff --git a/folder_paths.py b/folder_paths.pyindex ef9b8ccfa..f1bf40f8c 100644--- a/folder_paths.py+++ b/folder_paths.py@@ -29,6 +29,8 @@ folder_names_and_paths["custom_nodes"] = ([os.path.join(base_path, "custom_nodesfolder_names_and_paths["hypernetworks"] = ([os.path.join(models_dir, "hypernetworks")], supported_pt_extensions)+folder_names_and_paths["photomaker"] = ([os.path.join(models_dir, "photomaker")], supported_pt_extensions)+folder_names_and_paths["classifiers"] = ([os.path.join(models_dir, "classifiers")], {""})output_directory = os.path.join(os.path.dirname(os.path.realpath(__file__)), "output")diff --git a/models/photomaker/put_photomaker_models_here b/models/photomaker/put_photomaker_models_herenew file mode 100644index 000000000..e69de29bbdiff --git a/nodes.py b/nodes.pyindex 6c7317b69..4ad35f79b 100644--- a/nodes.py+++ b/nodes.py@@ -1943,6 +1943,7 @@ def init_custom_nodes():"nodes_perpneg.py","nodes_stable3d.py","nodes_sdupscale.py",+ "nodes_photomaker.py",]for node_file in extras_files: