The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Support for qwen edit plus model. Use the new TextEncodeQwenImageEditPlus. (#9986)
comfyanonymous1fee8827cb81
diff --git a/comfy/text_encoders/llama.py b/comfy/text_encoders/llama.pyindex 5e11956b5..c5a48ba9f 100644--- a/comfy/text_encoders/llama.py+++ b/comfy/text_encoders/llama.py@@ -400,21 +400,25 @@ class Qwen25_7BVLI(BaseLlama, torch.nn.Module):def forward(self, x, attention_mask=None, embeds=None, num_tokens=None, intermediate_output=None, final_layer_norm_intermediate=True, dtype=None, embeds_info=[]):grid = None+ position_ids = None+ offset = 0for e in embeds_info:if e.get("type") == "image":grid = e.get("extra", None)- position_ids = torch.zeros((3, embeds.shape[1]), device=embeds.device)start = e.get("index")- position_ids[:, :start] = torch.arange(0, start, device=embeds.device)+ if position_ids is None:+ position_ids = torch.zeros((3, embeds.shape[1]), device=embeds.device)+ position_ids[:, :start] = torch.arange(0, start, device=embeds.device)end = e.get("size") + startlen_max = int(grid.max()) // 2start_next = len_max + start- position_ids[:, end:] = torch.arange(start_next, start_next + (embeds.shape[1] - end), device=embeds.device)- position_ids[0, start:end] = start+ position_ids[:, end:] = torch.arange(start_next + offset, start_next + (embeds.shape[1] - end) + offset, device=embeds.device)+ position_ids[0, start:end] = start + offsetmax_d = int(grid[0][1]) // 2- position_ids[1, start:end] = torch.arange(start, start + max_d, device=embeds.device).unsqueeze(1).repeat(1, math.ceil((end - start) / max_d)).flatten(0)[:end - start]+ position_ids[1, start:end] = torch.arange(start + offset, start + max_d + offset, device=embeds.device).unsqueeze(1).repeat(1, math.ceil((end - start) / max_d)).flatten(0)[:end - start]max_d = int(grid[0][2]) // 2- position_ids[2, start:end] = torch.arange(start, start + max_d, device=embeds.device).unsqueeze(0).repeat(math.ceil((end - start) / max_d), 1).flatten(0)[:end - start]+ position_ids[2, start:end] = torch.arange(start + offset, start + max_d + offset, device=embeds.device).unsqueeze(0).repeat(math.ceil((end - start) / max_d), 1).flatten(0)[:end - start]+ offset += len_max - (end - start)if grid is None:position_ids = Nonediff --git a/comfy_extras/nodes_qwen.py b/comfy_extras/nodes_qwen.pyindex fff89556f..49747dc7a 100644--- a/comfy_extras/nodes_qwen.py+++ b/comfy_extras/nodes_qwen.py@@ -43,6 +43,61 @@ class TextEncodeQwenImageEdit:return (conditioning, )+class TextEncodeQwenImageEditPlus:+ @classmethod+ def INPUT_TYPES(s):+ return {"required": {+ "clip": ("CLIP", ),+ "prompt": ("STRING", {"multiline": True, "dynamicPrompts": True}),+ },+ "optional": {"vae": ("VAE", ),+ "image1": ("IMAGE", ),+ "image2": ("IMAGE", ),+ "image3": ("IMAGE", ),+ }}++ RETURN_TYPES = ("CONDITIONING",)+ FUNCTION = "encode"++ CATEGORY = "advanced/conditioning"++ def encode(self, clip, prompt, vae=None, image1=None, image2=None, image3=None):+ ref_latents = []+ images = [image1, image2, image3]+ images_vl = []+ llama_template = "<|im_start|>system\nDescribe the key features of the input image (color, shape, size, texture, objects, background), then explain how the user's text instruction should alter or modify the image. Generate a new image that meets the user's requirements while maintaining consistency with the original input where appropriate.<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n"+ image_prompt = ""++ for i, image in enumerate(images):+ if image is not None:+ samples = image.movedim(-1, 1)+ total = int(384 * 384)++ scale_by = math.sqrt(total / (samples.shape[3] * samples.shape[2]))+ width = round(samples.shape[3] * scale_by)+ height = round(samples.shape[2] * scale_by)++ s = comfy.utils.common_upscale(samples, width, height, "area", "disabled")+ images_vl.append(s.movedim(1, -1))+ if vae is not None:+ total = int(1024 * 1024)+ scale_by = math.sqrt(total / (samples.shape[3] * samples.shape[2]))+ width = round(samples.shape[3] * scale_by / 8.0) * 8+ height = round(samples.shape[2] * scale_by / 8.0) * 8++ s = comfy.utils.common_upscale(samples, width, height, "area", "disabled")+ ref_latents.append(vae.encode(s.movedim(1, -1)[:, :, :, :3]))++ image_prompt += "Picture {}: <|vision_start|><|image_pad|><|vision_end|>".format(i + 1)++ tokens = clip.tokenize(image_prompt + prompt, images=images_vl, llama_template=llama_template)+ conditioning = clip.encode_from_tokens_scheduled(tokens)+ if len(ref_latents) > 0:+ conditioning = node_helpers.conditioning_set_values(conditioning, {"reference_latents": ref_latents}, append=True)+ return (conditioning, )++NODE_CLASS_MAPPINGS = {"TextEncodeQwenImageEdit": TextEncodeQwenImageEdit,+ "TextEncodeQwenImageEditPlus": TextEncodeQwenImageEditPlus,}