Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Support for qwen edit plus model. Use the new TextEncodeQwenImageEditPlus. (#9986)

comfyanonymous1fee8827cb81
diff --git a/comfy/text_encoders/llama.py b/comfy/text_encoders/llama.py
index 5e11956b5..c5a48ba9f 100644
--- a/comfy/text_encoders/llama.py
+++ b/comfy/text_encoders/llama.py
@@ -400,21 +400,25 @@ class Qwen25_7BVLI(BaseLlama, torch.nn.Module):
def forward(self, x, attention_mask=None, embeds=None, num_tokens=None, intermediate_output=None, final_layer_norm_intermediate=True, dtype=None, embeds_info=[]):
grid = None
+ position_ids = None
+ offset = 0
for e in embeds_info:
if e.get("type") == "image":
grid = e.get("extra", None)
- position_ids = torch.zeros((3, embeds.shape[1]), device=embeds.device)
start = e.get("index")
- position_ids[:, :start] = torch.arange(0, start, device=embeds.device)
+ if position_ids is None:
+ position_ids = torch.zeros((3, embeds.shape[1]), device=embeds.device)
+ position_ids[:, :start] = torch.arange(0, start, device=embeds.device)
end = e.get("size") + start
len_max = int(grid.max()) // 2
start_next = len_max + start
- position_ids[:, end:] = torch.arange(start_next, start_next + (embeds.shape[1] - end), device=embeds.device)
- position_ids[0, start:end] = start
+ position_ids[:, end:] = torch.arange(start_next + offset, start_next + (embeds.shape[1] - end) + offset, device=embeds.device)
+ position_ids[0, start:end] = start + offset
max_d = int(grid[0][1]) // 2
- position_ids[1, start:end] = torch.arange(start, start + max_d, device=embeds.device).unsqueeze(1).repeat(1, math.ceil((end - start) / max_d)).flatten(0)[:end - start]
+ position_ids[1, start:end] = torch.arange(start + offset, start + max_d + offset, device=embeds.device).unsqueeze(1).repeat(1, math.ceil((end - start) / max_d)).flatten(0)[:end - start]
max_d = int(grid[0][2]) // 2
- position_ids[2, start:end] = torch.arange(start, start + max_d, device=embeds.device).unsqueeze(0).repeat(math.ceil((end - start) / max_d), 1).flatten(0)[:end - start]
+ position_ids[2, start:end] = torch.arange(start + offset, start + max_d + offset, device=embeds.device).unsqueeze(0).repeat(math.ceil((end - start) / max_d), 1).flatten(0)[:end - start]
+ offset += len_max - (end - start)
if grid is None:
position_ids = None
diff --git a/comfy_extras/nodes_qwen.py b/comfy_extras/nodes_qwen.py
index fff89556f..49747dc7a 100644
--- a/comfy_extras/nodes_qwen.py
+++ b/comfy_extras/nodes_qwen.py
@@ -43,6 +43,61 @@ class TextEncodeQwenImageEdit:
return (conditioning, )
+class TextEncodeQwenImageEditPlus:
+ @classmethod
+ def INPUT_TYPES(s):
+ return {"required": {
+ "clip": ("CLIP", ),
+ "prompt": ("STRING", {"multiline": True, "dynamicPrompts": True}),
+ },
+ "optional": {"vae": ("VAE", ),
+ "image1": ("IMAGE", ),
+ "image2": ("IMAGE", ),
+ "image3": ("IMAGE", ),
+ }}
+
+ RETURN_TYPES = ("CONDITIONING",)
+ FUNCTION = "encode"
+
+ CATEGORY = "advanced/conditioning"
+
+ def encode(self, clip, prompt, vae=None, image1=None, image2=None, image3=None):
+ ref_latents = []
+ images = [image1, image2, image3]
+ images_vl = []
+ llama_template = "<|im_start|>system\nDescribe the key features of the input image (color, shape, size, texture, objects, background), then explain how the user's text instruction should alter or modify the image. Generate a new image that meets the user's requirements while maintaining consistency with the original input where appropriate.<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n"
+ image_prompt = ""
+
+ for i, image in enumerate(images):
+ if image is not None:
+ samples = image.movedim(-1, 1)
+ total = int(384 * 384)
+
+ scale_by = math.sqrt(total / (samples.shape[3] * samples.shape[2]))
+ width = round(samples.shape[3] * scale_by)
+ height = round(samples.shape[2] * scale_by)
+
+ s = comfy.utils.common_upscale(samples, width, height, "area", "disabled")
+ images_vl.append(s.movedim(1, -1))
+ if vae is not None:
+ total = int(1024 * 1024)
+ scale_by = math.sqrt(total / (samples.shape[3] * samples.shape[2]))
+ width = round(samples.shape[3] * scale_by / 8.0) * 8
+ height = round(samples.shape[2] * scale_by / 8.0) * 8
+
+ s = comfy.utils.common_upscale(samples, width, height, "area", "disabled")
+ ref_latents.append(vae.encode(s.movedim(1, -1)[:, :, :, :3]))
+
+ image_prompt += "Picture {}: <|vision_start|><|image_pad|><|vision_end|>".format(i + 1)
+
+ tokens = clip.tokenize(image_prompt + prompt, images=images_vl, llama_template=llama_template)
+ conditioning = clip.encode_from_tokens_scheduled(tokens)
+ if len(ref_latents) > 0:
+ conditioning = node_helpers.conditioning_set_values(conditioning, {"reference_latents": ref_latents}, append=True)
+ return (conditioning, )
+
+
NODE_CLASS_MAPPINGS = {
"TextEncodeQwenImageEdit": TextEncodeQwenImageEdit,
+ "TextEncodeQwenImageEditPlus": TextEncodeQwenImageEditPlus,
}