The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Initial support for qwen image model. (#9179)
comfyanonymousc012400240d4
diff --git a/comfy/ldm/qwen_image/model.py b/comfy/ldm/qwen_image/model.pynew file mode 100644index 000000000..ff631a60f--- /dev/null+++ b/comfy/ldm/qwen_image/model.py@@ -0,0 +1,399 @@+# https://github.com/QwenLM/Qwen-Image (Apache 2.0)+import torch+import torch.nn as nn+import torch.nn.functional as F+from typing import Optional, Tuple+from einops import repeat++from comfy.ldm.lightricks.model import TimestepEmbedding, Timesteps+from comfy.ldm.modules.attention import optimized_attention_masked+from comfy.ldm.flux.layers import EmbedND+++class GELU(nn.Module):+ def __init__(self, dim_in: int, dim_out: int, approximate: str = "none", bias: bool = True, dtype=None, device=None, operations=None):+ super().__init__()+ self.proj = operations.Linear(dim_in, dim_out, bias=bias, dtype=dtype, device=device)+ self.approximate = approximate++ def forward(self, hidden_states):+ hidden_states = self.proj(hidden_states)+ hidden_states = F.gelu(hidden_states, approximate=self.approximate)+ return hidden_states+++class FeedForward(nn.Module):+ def __init__(+ self,+ dim: int,+ dim_out: Optional[int] = None,+ mult: int = 4,+ dropout: float = 0.0,+ inner_dim=None,+ bias: bool = True,+ dtype=None, device=None, operations=None+ ):+ super().__init__()+ if inner_dim is None:+ inner_dim = int(dim * mult)+ dim_out = dim_out if dim_out is not None else dim++ self.net = nn.ModuleList([])+ self.net.append(GELU(dim, inner_dim, approximate="tanh", bias=bias, dtype=dtype, device=device, operations=operations))+ self.net.append(nn.Dropout(dropout))+ self.net.append(operations.Linear(inner_dim, dim_out, bias=bias, dtype=dtype, device=device))++ def forward(self, hidden_states: torch.Tensor, *args, **kwargs) -> torch.Tensor:+ for module in self.net:+ hidden_states = module(hidden_states)+ return hidden_states+++def apply_rotary_emb(x, freqs_cis):+ if x.shape[1] == 0:+ return x++ t_ = x.reshape(*x.shape[:-1], -1, 1, 2)+ t_out = freqs_cis[..., 0] * t_[..., 0] + freqs_cis[..., 1] * t_[..., 1]+ return t_out.reshape(*x.shape)+++class QwenTimestepProjEmbeddings(nn.Module):+ def __init__(self, embedding_dim, pooled_projection_dim, dtype=None, device=None, operations=None):+ super().__init__()+ self.time_proj = Timesteps(num_channels=256, flip_sin_to_cos=True, downscale_freq_shift=0, scale=1000)+ self.timestep_embedder = TimestepEmbedding(+ in_channels=256,+ time_embed_dim=embedding_dim,+ dtype=dtype,+ device=device,+ operations=operations+ )++ def forward(self, timestep, hidden_states):+ timesteps_proj = self.time_proj(timestep)+ timesteps_emb = self.timestep_embedder(timesteps_proj.to(dtype=hidden_states.dtype))+ return timesteps_emb+++class Attention(nn.Module):+ def __init__(+ self,+ query_dim: int,+ dim_head: int = 64,+ heads: int = 8,+ dropout: float = 0.0,+ bias: bool = False,+ eps: float = 1e-5,+ out_bias: bool = True,+ out_dim: int = None,+ out_context_dim: int = None,+ dtype=None,+ device=None,+ operations=None+ ):+ super().__init__()+ self.inner_dim = out_dim if out_dim is not None else dim_head * heads+ self.inner_kv_dim = self.inner_dim+ self.heads = heads+ self.dim_head = dim_head+ self.out_dim = out_dim if out_dim is not None else query_dim+ self.out_context_dim = out_context_dim if out_context_dim is not None else query_dim+ self.dropout = dropout++ # Q/K normalization+ self.norm_q = operations.RMSNorm(dim_head, eps=eps, elementwise_affine=True, dtype=dtype, device=device)+ self.norm_k = operations.RMSNorm(dim_head, eps=eps, elementwise_affine=True, dtype=dtype, device=device)+ self.norm_added_q = operations.RMSNorm(dim_head, eps=eps, dtype=dtype, device=device)+ self.norm_added_k = operations.RMSNorm(dim_head, eps=eps, dtype=dtype, device=device)++ # Image stream projections+ self.to_q = operations.Linear(query_dim, self.inner_dim, bias=bias, dtype=dtype, device=device)+ self.to_k = operations.Linear(query_dim, self.inner_kv_dim, bias=bias, dtype=dtype, device=device)+ self.to_v = operations.Linear(query_dim, self.inner_kv_dim, bias=bias, dtype=dtype, device=device)++ # Text stream projections+ self.add_q_proj = operations.Linear(query_dim, self.inner_dim, bias=bias, dtype=dtype, device=device)+ self.add_k_proj = operations.Linear(query_dim, self.inner_kv_dim, bias=bias, dtype=dtype, device=device)+ self.add_v_proj = operations.Linear(query_dim, self.inner_kv_dim, bias=bias, dtype=dtype, device=device)++ # Output projections+ self.to_out = nn.ModuleList([+ operations.Linear(self.inner_dim, self.out_dim, bias=out_bias, dtype=dtype, device=device),+ nn.Dropout(dropout)+ ])+ self.to_add_out = operations.Linear(self.inner_dim, self.out_context_dim, bias=out_bias, dtype=dtype, device=device)++ def forward(+ self,+ hidden_states: torch.FloatTensor, # Image stream+ encoder_hidden_states: torch.FloatTensor = None, # Text stream+ encoder_hidden_states_mask: torch.FloatTensor = None,+ attention_mask: Optional[torch.FloatTensor] = None,+ image_rotary_emb: Optional[torch.Tensor] = None,+ ) -> Tuple[torch.Tensor, torch.Tensor]:+ seq_txt = encoder_hidden_states.shape[1]++ img_query = self.to_q(hidden_states).unflatten(-1, (self.heads, -1))+ img_key = self.to_k(hidden_states).unflatten(-1, (self.heads, -1))+ img_value = self.to_v(hidden_states).unflatten(-1, (self.heads, -1))++ txt_query = self.add_q_proj(encoder_hidden_states).unflatten(-1, (self.heads, -1))+ txt_key = self.add_k_proj(encoder_hidden_states).unflatten(-1, (self.heads, -1))+ txt_value = self.add_v_proj(encoder_hidden_states).unflatten(-1, (self.heads, -1))++ img_query = self.norm_q(img_query)+ img_key = self.norm_k(img_key)+ txt_query = self.norm_added_q(txt_query)+ txt_key = self.norm_added_k(txt_key)++ joint_query = torch.cat([txt_query, img_query], dim=1)+ joint_key = torch.cat([txt_key, img_key], dim=1)+ joint_value = torch.cat([txt_value, img_value], dim=1)++ joint_query = apply_rotary_emb(joint_query, image_rotary_emb)+ joint_key = apply_rotary_emb(joint_key, image_rotary_emb)++ joint_query = joint_query.flatten(start_dim=2)+ joint_key = joint_key.flatten(start_dim=2)+ joint_value = joint_value.flatten(start_dim=2)++ joint_hidden_states = optimized_attention_masked(joint_query, joint_key, joint_value, self.heads, attention_mask)++ txt_attn_output = joint_hidden_states[:, :seq_txt, :]+ img_attn_output = joint_hidden_states[:, seq_txt:, :]++ img_attn_output = self.to_out[0](img_attn_output)+ img_attn_output = self.to_out[1](img_attn_output)+ txt_attn_output = self.to_add_out(txt_attn_output)++ return img_attn_output, txt_attn_output+++class QwenImageTransformerBlock(nn.Module):+ def __init__(+ self,+ dim: int,+ num_attention_heads: int,+ attention_head_dim: int,+ eps: float = 1e-6,+ dtype=None,+ device=None,+ operations=None+ ):+ super().__init__()+ self.dim = dim+ self.num_attention_heads = num_attention_heads+ self.attention_head_dim = attention_head_dim++ self.img_mod = nn.Sequential(+ nn.SiLU(),+ operations.Linear(dim, 6 * dim, bias=True, dtype=dtype, device=device),+ )+ self.img_norm1 = operations.LayerNorm(dim, elementwise_affine=False, eps=eps, dtype=dtype, device=device)+ self.img_norm2 = operations.LayerNorm(dim, elementwise_affine=False, eps=eps, dtype=dtype, device=device)+ self.img_mlp = FeedForward(dim=dim, dim_out=dim, dtype=dtype, device=device, operations=operations)++ self.txt_mod = nn.Sequential(+ nn.SiLU(),+ operations.Linear(dim, 6 * dim, bias=True, dtype=dtype, device=device),+ )+ self.txt_norm1 = operations.LayerNorm(dim, elementwise_affine=False, eps=eps, dtype=dtype, device=device)+ self.txt_norm2 = operations.LayerNorm(dim, elementwise_affine=False, eps=eps, dtype=dtype, device=device)+ self.txt_mlp = FeedForward(dim=dim, dim_out=dim, dtype=dtype, device=device, operations=operations)++ self.attn = Attention(+ query_dim=dim,+ dim_head=attention_head_dim,+ heads=num_attention_heads,+ out_dim=dim,+ bias=True,+ eps=eps,+ dtype=dtype,+ device=device,+ operations=operations,+ )++ def _modulate(self, x, mod_params):+ shift, scale, gate = mod_params.chunk(3, dim=-1)+ return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1), gate.unsqueeze(1)++ def forward(+ self,+ hidden_states: torch.Tensor,+ encoder_hidden_states: torch.Tensor,+ encoder_hidden_states_mask: torch.Tensor,+ temb: torch.Tensor,+ image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,+ ) -> Tuple[torch.Tensor, torch.Tensor]:+ img_mod_params = self.img_mod(temb)+ txt_mod_params = self.txt_mod(temb)+ img_mod1, img_mod2 = img_mod_params.chunk(2, dim=-1)+ txt_mod1, txt_mod2 = txt_mod_params.chunk(2, dim=-1)++ img_normed = self.img_norm1(hidden_states)+ img_modulated, img_gate1 = self._modulate(img_normed, img_mod1)+ txt_normed = self.txt_norm1(encoder_hidden_states)+ txt_modulated, txt_gate1 = self._modulate(txt_normed, txt_mod1)++ img_attn_output, txt_attn_output = self.attn(+ hidden_states=img_modulated,+ encoder_hidden_states=txt_modulated,+ encoder_hidden_states_mask=encoder_hidden_states_mask,+ image_rotary_emb=image_rotary_emb,+ )++ hidden_states = hidden_states + img_gate1 * img_attn_output+ encoder_hidden_states = encoder_hidden_states + txt_gate1 * txt_attn_output++ img_normed2 = self.img_norm2(hidden_states)+ img_modulated2, img_gate2 = self._modulate(img_normed2, img_mod2)+ hidden_states = hidden_states + img_gate2 * self.img_mlp(img_modulated2)++ txt_normed2 = self.txt_norm2(encoder_hidden_states)+ txt_modulated2, txt_gate2 = self._modulate(txt_normed2, txt_mod2)+ encoder_hidden_states = encoder_hidden_states + txt_gate2 * self.txt_mlp(txt_modulated2)++ return encoder_hidden_states, hidden_states+++class LastLayer(nn.Module):+ def __init__(+ self,+ embedding_dim: int,+ conditioning_embedding_dim: int,+ elementwise_affine=False,+ eps=1e-6,+ bias=True,+ dtype=None, device=None, operations=None+ ):+ super().__init__()+ self.silu = nn.SiLU()+ self.linear = operations.Linear(conditioning_embedding_dim, embedding_dim * 2, bias=bias, dtype=dtype, device=device)+ self.norm = operations.LayerNorm(embedding_dim, eps, elementwise_affine=False, bias=bias, dtype=dtype, device=device)++ def forward(self, x: torch.Tensor, conditioning_embedding: torch.Tensor) -> torch.Tensor:+ emb = self.linear(self.silu(conditioning_embedding))+ scale, shift = torch.chunk(emb, 2, dim=1)+ x = self.norm(x) * (1 + scale)[:, None, :] + shift[:, None, :]+ return x+++class QwenImageTransformer2DModel(nn.Module):+ def __init__(+ self,+ patch_size: int = 2,+ in_channels: int = 64,+ out_channels: Optional[int] = 16,+ num_layers: int = 60,+ attention_head_dim: int = 128,+ num_attention_heads: int = 24,+ joint_attention_dim: int = 3584,+ pooled_projection_dim: int = 768,+ guidance_embeds: bool = False,+ axes_dims_rope: Tuple[int, int, int] = (16, 56, 56),+ image_model=None,+ dtype=None,+ device=None,+ operations=None,+ ):+ super().__init__()+ self.dtype = dtype+ self.patch_size = patch_size+ self.out_channels = out_channels or in_channels+ self.inner_dim = num_attention_heads * attention_head_dim++ self.pe_embedder = EmbedND(dim=attention_head_dim, theta=10000, axes_dim=list(axes_dims_rope))++ self.time_text_embed = QwenTimestepProjEmbeddings(+ embedding_dim=self.inner_dim,+ pooled_projection_dim=pooled_projection_dim,+ dtype=dtype,+ device=device,+ operations=operations+ )++ self.txt_norm = operations.RMSNorm(joint_attention_dim, eps=1e-6, dtype=dtype, device=device)+ self.img_in = operations.Linear(in_channels, self.inner_dim, dtype=dtype, device=device)+ self.txt_in = operations.Linear(joint_attention_dim, self.inner_dim, dtype=dtype, device=device)++ self.transformer_blocks = nn.ModuleList([+ QwenImageTransformerBlock(+ dim=self.inner_dim,+ num_attention_heads=num_attention_heads,+ attention_head_dim=attention_head_dim,+ dtype=dtype,+ device=device,+ operations=operations+ )+ for _ in range(num_layers)+ ])++ self.norm_out = LastLayer(self.inner_dim, self.inner_dim, dtype=dtype, device=device, operations=operations)+ self.proj_out = operations.Linear(self.inner_dim, patch_size * patch_size * self.out_channels, bias=True, dtype=dtype, device=device)+ self.gradient_checkpointing = False++ def pos_embeds(self, x, context):+ bs, c, t, h, w = x.shape+ patch_size = self.patch_size+ h_len = ((h + (patch_size // 2)) // patch_size)+ w_len = ((w + (patch_size // 2)) // patch_size)++ img_ids = torch.zeros((h_len, w_len, 3), device=x.device, dtype=x.dtype)+ img_ids[:, :, 1] = img_ids[:, :, 1] + torch.linspace(0, h_len - 1, steps=h_len, device=x.device, dtype=x.dtype).unsqueeze(1)+ img_ids[:, :, 2] = img_ids[:, :, 2] + torch.linspace(0, w_len - 1, steps=w_len, device=x.device, dtype=x.dtype).unsqueeze(0)+ img_ids = repeat(img_ids, "h w c -> b (h w) c", b=bs)++ txt_start = round(max(h_len, w_len))+ txt_ids = torch.linspace(txt_start, txt_start + context.shape[1], steps=context.shape[1], device=x.device, dtype=x.dtype).reshape(1, -1, 1).repeat(bs, 1, 3)+ ids = torch.cat((txt_ids, img_ids), dim=1)+ return self.pe_embedder(ids).squeeze(1).unsqueeze(2).to(x.dtype)++ def forward(+ self,+ x,+ timesteps,+ context,+ attention_mask=None,+ guidance: torch.Tensor = None,+ **kwargs+ ):+ timestep = timesteps+ encoder_hidden_states = context+ encoder_hidden_states_mask = attention_mask++ image_rotary_emb = self.pos_embeds(x, context)++ orig_shape = x.shape+ hidden_states = x.view(orig_shape[0], orig_shape[1], orig_shape[-2] // 2, 2, orig_shape[-1] // 2, 2)+ hidden_states = hidden_states.permute(0, 2, 4, 1, 3, 5)+ hidden_states = hidden_states.reshape(orig_shape[0], (orig_shape[-2] // 2) * (orig_shape[-1] // 2), orig_shape[1] * 4)++ hidden_states = self.img_in(hidden_states)+ encoder_hidden_states = self.txt_norm(encoder_hidden_states)+ encoder_hidden_states = self.txt_in(encoder_hidden_states)++ if guidance is not None:+ guidance = guidance * 1000++ temb = (+ self.time_text_embed(timestep, hidden_states)+ if guidance is None+ else self.time_text_embed(timestep, guidance, hidden_states)+ )++ for block in self.transformer_blocks:+ encoder_hidden_states, hidden_states = block(+ hidden_states=hidden_states,+ encoder_hidden_states=encoder_hidden_states,+ encoder_hidden_states_mask=encoder_hidden_states_mask,+ temb=temb,+ image_rotary_emb=image_rotary_emb,+ )++ hidden_states = self.norm_out(hidden_states, temb)+ hidden_states = self.proj_out(hidden_states)++ hidden_states = hidden_states.view(orig_shape[0], orig_shape[-2] // 2, orig_shape[-1] // 2, orig_shape[1], 2, 2)+ hidden_states = hidden_states.permute(0, 3, 1, 4, 2, 5)+ return hidden_states.reshape(orig_shape)diff --git a/comfy/model_base.py b/comfy/model_base.pyindex a06686436..8a2d9cbe6 100644--- a/comfy/model_base.py+++ b/comfy/model_base.py@@ -42,6 +42,7 @@ import comfy.ldm.hidream.modelimport comfy.ldm.chroma.modelimport comfy.ldm.ace.modelimport comfy.ldm.omnigen.omnigen2+import comfy.ldm.qwen_image.modelimport comfy.model_managementimport comfy.patcher_extension@@ -1308,3 +1309,14 @@ class Omnigen2(BaseModel):if ref_latents is not None:out['ref_latents'] = list([1, 16, sum(map(lambda a: math.prod(a.size()), ref_latents)) // 16])return out++class QwenImage(BaseModel):+ def __init__(self, model_config, model_type=ModelType.FLUX, device=None):+ super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.qwen_image.model.QwenImageTransformer2DModel)++ def extra_conds(self, **kwargs):+ out = super().extra_conds(**kwargs)+ cross_attn = kwargs.get("cross_attn", None)+ if cross_attn is not None:+ out['c_crossattn'] = comfy.conds.CONDRegular(cross_attn)+ return outdiff --git a/comfy/model_detection.py b/comfy/model_detection.pyindex 9fc1f42de..8b57ebd2f 100644--- a/comfy/model_detection.py+++ b/comfy/model_detection.py@@ -481,6 +481,11 @@ def detect_unet_config(state_dict, key_prefix, metadata=None):dit_config["timestep_scale"] = 1000.0return dit_config+ if '{}txt_norm.weight'.format(key_prefix) in state_dict_keys: # Qwen Image+ dit_config = {}+ dit_config["image_model"] = "qwen_image"+ return dit_config+if '{}input_blocks.0.0.weight'.format(key_prefix) not in state_dict_keys:return None@@ -867,7 +872,7 @@ def convert_diffusers_mmdit(state_dict, output_prefix=""):depth_single_blocks = count_blocks(state_dict, 'single_transformer_blocks.{}.')hidden_size = state_dict["x_embedder.bias"].shape[0]sd_map = comfy.utils.flux_to_diffusers({"depth": depth, "depth_single_blocks": depth_single_blocks, "hidden_size": hidden_size}, output_prefix=output_prefix)- elif 'transformer_blocks.0.attn.add_q_proj.weight' in state_dict: #SD3+ elif 'transformer_blocks.0.attn.add_q_proj.weight' in state_dict and 'pos_embed.proj.weight' in state_dict: #SD3num_blocks = count_blocks(state_dict, 'transformer_blocks.{}.')depth = state_dict["pos_embed.proj.weight"].shape[0] // 64sd_map = comfy.utils.mmdit_to_diffusers({"depth": depth, "num_blocks": num_blocks}, output_prefix=output_prefix)diff --git a/comfy/sd.py b/comfy/sd.pyindex e0498e585..bb5d61fb3 100644--- a/comfy/sd.py+++ b/comfy/sd.py@@ -47,6 +47,7 @@ import comfy.text_encoders.wanimport comfy.text_encoders.hidreamimport comfy.text_encoders.aceimport comfy.text_encoders.omnigen2+import comfy.text_encoders.qwen_imageimport comfy.model_patcherimport comfy.lora@@ -771,6 +772,7 @@ class CLIPType(Enum):CHROMA = 15ACE = 16OMNIGEN2 = 17+ QWEN_IMAGE = 18def load_clip(ckpt_paths, embedding_directory=None, clip_type=CLIPType.STABLE_DIFFUSION, model_options={}):@@ -791,6 +793,7 @@ class TEModel(Enum):T5_XXL_OLD = 8GEMMA_2_2B = 9QWEN25_3B = 10+ QWEN25_7B = 11def detect_te_model(sd):if "text_model.encoder.layers.30.mlp.fc1.weight" in sd:@@ -812,7 +815,11 @@ def detect_te_model(sd):if 'model.layers.0.post_feedforward_layernorm.weight' in sd:return TEModel.GEMMA_2_2Bif 'model.layers.0.self_attn.k_proj.bias' in sd:- return TEModel.QWEN25_3B+ weight = sd['model.layers.0.self_attn.k_proj.bias']+ if weight.shape[0] == 256:+ return TEModel.QWEN25_3B+ if weight.shape[0] == 512:+ return TEModel.QWEN25_7Bif "model.layers.0.post_attention_layernorm.weight" in sd:return TEModel.LLAMA3_8return None@@ -917,6 +924,9 @@ def load_text_encoder_state_dicts(state_dicts=[], embedding_directory=None, clipelif te_model == TEModel.QWEN25_3B:clip_target.clip = comfy.text_encoders.omnigen2.te(**llama_detect(clip_data))clip_target.tokenizer = comfy.text_encoders.omnigen2.Omnigen2Tokenizer+ elif te_model == TEModel.QWEN25_7B:+ clip_target.clip = comfy.text_encoders.qwen_image.te(**llama_detect(clip_data))+ clip_target.tokenizer = comfy.text_encoders.qwen_image.QwenImageTokenizerelse:# clip_lif clip_type == CLIPType.SD3:diff --git a/comfy/supported_models.py b/comfy/supported_models.pyindex 8f3f4652d..880055bd3 100644--- a/comfy/supported_models.py+++ b/comfy/supported_models.py@@ -19,6 +19,7 @@ import comfy.text_encoders.lumina2import comfy.text_encoders.wanimport comfy.text_encoders.aceimport comfy.text_encoders.omnigen2+import comfy.text_encoders.qwen_imagefrom . import supported_models_basefrom . import latent_formats@@ -1229,7 +1230,36 @@ class Omnigen2(supported_models_base.BASE):hunyuan_detect = comfy.text_encoders.hunyuan_video.llama_detect(state_dict, "{}qwen25_3b.transformer.".format(pref))return supported_models_base.ClipTarget(comfy.text_encoders.omnigen2.Omnigen2Tokenizer, comfy.text_encoders.omnigen2.te(**hunyuan_detect))+class QwenImage(supported_models_base.BASE):+ unet_config = {+ "image_model": "qwen_image",+ }++ sampling_settings = {+ "multiplier": 1.0,+ "shift": 2.6,+ }++ memory_usage_factor = 1.8 #TODO++ unet_extra_config = {}+ latent_format = latent_formats.Wan21++ supported_inference_dtypes = [torch.bfloat16, torch.float32]++ vae_key_prefix = ["vae."]+ text_encoder_key_prefix = ["text_encoders."]++ def get_model(self, state_dict, prefix="", device=None):+ out = model_base.QwenImage(self, device=device)+ return out++ def clip_target(self, state_dict={}):+ pref = self.text_encoder_key_prefix[0]+ hunyuan_detect = comfy.text_encoders.hunyuan_video.llama_detect(state_dict, "{}qwen25_7b.transformer.".format(pref))+ return supported_models_base.ClipTarget(comfy.text_encoders.qwen_image.QwenImageTokenizer, comfy.text_encoders.qwen_image.te(**hunyuan_detect))+-models = [LotusD, Stable_Zero123, SD15_instructpix2pix, SD15, SD20, SD21UnclipL, SD21UnclipH, SDXL_instructpix2pix, SDXLRefiner, SDXL, SSD1B, KOALA_700M, KOALA_1B, Segmind_Vega, SD_X4Upscaler, Stable_Cascade_C, Stable_Cascade_B, SV3D_u, SV3D_p, SD3, StableAudio, AuraFlow, PixArtAlpha, PixArtSigma, HunyuanDiT, HunyuanDiT1, FluxInpaint, Flux, FluxSchnell, GenmoMochi, LTXV, HunyuanVideoSkyreelsI2V, HunyuanVideoI2V, HunyuanVideo, CosmosT2V, CosmosI2V, CosmosT2IPredict2, CosmosI2VPredict2, Lumina2, WAN22_T2V, WAN21_T2V, WAN21_I2V, WAN21_FunControl2V, WAN21_Vace, WAN21_Camera, Hunyuan3Dv2mini, Hunyuan3Dv2, HiDream, Chroma, ACEStep, Omnigen2]+models = [LotusD, Stable_Zero123, SD15_instructpix2pix, SD15, SD20, SD21UnclipL, SD21UnclipH, SDXL_instructpix2pix, SDXLRefiner, SDXL, SSD1B, KOALA_700M, KOALA_1B, Segmind_Vega, SD_X4Upscaler, Stable_Cascade_C, Stable_Cascade_B, SV3D_u, SV3D_p, SD3, StableAudio, AuraFlow, PixArtAlpha, PixArtSigma, HunyuanDiT, HunyuanDiT1, FluxInpaint, Flux, FluxSchnell, GenmoMochi, LTXV, HunyuanVideoSkyreelsI2V, HunyuanVideoI2V, HunyuanVideo, CosmosT2V, CosmosI2V, CosmosT2IPredict2, CosmosI2VPredict2, Lumina2, WAN22_T2V, WAN21_T2V, WAN21_I2V, WAN21_FunControl2V, WAN21_Vace, WAN21_Camera, Hunyuan3Dv2mini, Hunyuan3Dv2, HiDream, Chroma, ACEStep, Omnigen2, QwenImage]models += [SVD_img2vid]diff --git a/comfy/text_encoders/llama.py b/comfy/text_encoders/llama.pyindex 7fbd0f604..1da6a0c94 100644--- a/comfy/text_encoders/llama.py+++ b/comfy/text_encoders/llama.py@@ -43,6 +43,23 @@ class Qwen25_3BConfig:mlp_activation = "silu"qkv_bias = True+@dataclass+class Qwen25_7BVLI_Config:+ vocab_size: int = 152064+ hidden_size: int = 3584+ intermediate_size: int = 18944+ num_hidden_layers: int = 28+ num_attention_heads: int = 28+ num_key_value_heads: int = 4+ max_position_embeddings: int = 128000+ rms_norm_eps: float = 1e-6+ rope_theta: float = 1000000.0+ transformer_type: str = "llama"+ head_dim = 128+ rms_norm_add = False+ mlp_activation = "silu"+ qkv_bias = True+@dataclassclass Gemma2_2B_Config:vocab_size: int = 256000@@ -348,6 +365,15 @@ class Qwen25_3B(BaseLlama, torch.nn.Module):self.model = Llama2_(config, device=device, dtype=dtype, ops=operations)self.dtype = dtype+class Qwen25_7BVLI(BaseLlama, torch.nn.Module):+ def __init__(self, config_dict, dtype, device, operations):+ super().__init__()+ config = Qwen25_7BVLI_Config(**config_dict)+ self.num_layers = config.num_hidden_layers++ self.model = Llama2_(config, device=device, dtype=dtype, ops=operations)+ self.dtype = dtype+class Gemma2_2B(BaseLlama, torch.nn.Module):def __init__(self, config_dict, dtype, device, operations):super().__init__()diff --git a/comfy/text_encoders/qwen_image.py b/comfy/text_encoders/qwen_image.pynew file mode 100644index 000000000..ce5c98097--- /dev/null+++ b/comfy/text_encoders/qwen_image.py@@ -0,0 +1,71 @@+from transformers import Qwen2Tokenizer+from comfy import sd1_clip+import comfy.text_encoders.llama+import os+import torch+import numbers++class Qwen25_7BVLITokenizer(sd1_clip.SDTokenizer):+ def __init__(self, embedding_directory=None, tokenizer_data={}):+ tokenizer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "qwen25_tokenizer")+ super().__init__(tokenizer_path, pad_with_end=False, embedding_size=3584, embedding_key='qwen25_7b', tokenizer_class=Qwen2Tokenizer, has_start_token=False, has_end_token=False, pad_to_max_length=False, max_length=99999999, min_length=1, pad_token=151643, tokenizer_data=tokenizer_data)+++class QwenImageTokenizer(sd1_clip.SD1Tokenizer):+ def __init__(self, embedding_directory=None, tokenizer_data={}):+ super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, name="qwen25_7b", tokenizer=Qwen25_7BVLITokenizer)+ self.llama_template = "<|im_start|>system\nDescribe the image by detailing the color, shape, size, texture, quantity, text, spatial relationships of the objects and background:<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n"++ def tokenize_with_weights(self, text, return_word_ids=False, llama_template=None,**kwargs):+ if llama_template is None:+ llama_text = self.llama_template.format(text)+ else:+ llama_text = llama_template.format(text)+ return super().tokenize_with_weights(llama_text, return_word_ids=return_word_ids, **kwargs)+++class Qwen25_7BVLIModel(sd1_clip.SDClipModel):+ def __init__(self, device="cpu", layer="last", layer_idx=None, dtype=None, attention_mask=True, model_options={}):+ super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config={}, dtype=dtype, special_tokens={"pad": 151643}, layer_norm_hidden_state=False, model_class=comfy.text_encoders.llama.Qwen25_7BVLI, enable_attention_masks=attention_mask, return_attention_masks=attention_mask, model_options=model_options)+++class QwenImageTEModel(sd1_clip.SD1ClipModel):+ def __init__(self, device="cpu", dtype=None, model_options={}):+ super().__init__(device=device, dtype=dtype, name="qwen25_7b", clip_model=Qwen25_7BVLIModel, model_options=model_options)++ def encode_token_weights(self, token_weight_pairs):+ out, pooled, extra = super().encode_token_weights(token_weight_pairs)+ tok_pairs = token_weight_pairs["qwen25_7b"][0]+ count_im_start = 0+ for i, v in enumerate(tok_pairs):+ elem = v[0]+ if not torch.is_tensor(elem):+ if isinstance(elem, numbers.Integral):+ if elem == 151644 and count_im_start < 2:+ template_end = i+ count_im_start += 1++ if out.shape[1] > (template_end + 3):+ if tok_pairs[template_end + 1][0] == 872:+ if tok_pairs[template_end + 2][0] == 198:+ template_end += 3++ out = out[:, template_end:]++ extra["attention_mask"] = extra["attention_mask"][:, template_end:]+ if extra["attention_mask"].sum() == torch.numel(extra["attention_mask"]):+ extra.pop("attention_mask") # attention mask is useless if no masked elements++ return out, pooled, extra+++def te(dtype_llama=None, llama_scaled_fp8=None):+ class QwenImageTEModel_(QwenImageTEModel):+ def __init__(self, device="cpu", dtype=None, model_options={}):+ if llama_scaled_fp8 is not None and "scaled_fp8" not in model_options:+ model_options = model_options.copy()+ model_options["scaled_fp8"] = llama_scaled_fp8+ if dtype_llama is not None:+ dtype = dtype_llama+ super().__init__(device=device, dtype=dtype, model_options=model_options)+ return QwenImageTEModel_diff --git a/nodes.py b/nodes.pyindex da4a46366..9bedbcaca 100644--- a/nodes.py+++ b/nodes.py@@ -925,7 +925,7 @@ class CLIPLoader:@classmethoddef INPUT_TYPES(s):return {"required": { "clip_name": (folder_paths.get_filename_list("text_encoders"), ),- "type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi", "ltxv", "pixart", "cosmos", "lumina2", "wan", "hidream", "chroma", "ace", "omnigen2"], ),+ "type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi", "ltxv", "pixart", "cosmos", "lumina2", "wan", "hidream", "chroma", "ace", "omnigen2", "qwen_image"], ),},"optional": {"device": (["default", "cpu"], {"advanced": True}),