The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Support zimage omni base model. (#11979)
comfyanonymous2108167f9f70
diff --git a/comfy/ldm/lumina/model.py b/comfy/ldm/lumina/model.pyindex afbab2ac7..139f879a1 100644--- a/comfy/ldm/lumina/model.py+++ b/comfy/ldm/lumina/model.py@@ -13,10 +13,53 @@ from comfy.ldm.modules.attention import optimized_attention_maskedfrom comfy.ldm.flux.layers import EmbedNDfrom comfy.ldm.flux.math import apply_ropeimport comfy.patcher_extension+import comfy.utils-def modulate(x, scale):- return x * (1 + scale.unsqueeze(1))+def invert_slices(slices, length):+ sorted_slices = sorted(slices)+ result = []+ current = 0++ for start, end in sorted_slices:+ if current < start:+ result.append((current, start))+ current = max(current, end)++ if current < length:+ result.append((current, length))++ return result+++def modulate(x, scale, timestep_zero_index=None):+ if timestep_zero_index is None:+ return x * (1 + scale.unsqueeze(1))+ else:+ scale = (1 + scale.unsqueeze(1))+ actual_batch = scale.size(0) // 2+ slices = timestep_zero_index+ invert = invert_slices(timestep_zero_index, x.shape[1])+ for s in slices:+ x[:, s[0]:s[1]] *= scale[actual_batch:]+ for s in invert:+ x[:, s[0]:s[1]] *= scale[:actual_batch]+ return x+++def apply_gate(gate, x, timestep_zero_index=None):+ if timestep_zero_index is None:+ return gate * x+ else:+ actual_batch = gate.size(0) // 2++ slices = timestep_zero_index+ invert = invert_slices(timestep_zero_index, x.shape[1])+ for s in slices:+ x[:, s[0]:s[1]] *= gate[actual_batch:]+ for s in invert:+ x[:, s[0]:s[1]] *= gate[:actual_batch]+ return x############################################################################## Core NextDiT Model #@@ -258,6 +301,7 @@ class JointTransformerBlock(nn.Module):x_mask: torch.Tensor,freqs_cis: torch.Tensor,adaln_input: Optional[torch.Tensor]=None,+ timestep_zero_index=None,transformer_options={},):"""@@ -276,18 +320,18 @@ class JointTransformerBlock(nn.Module):assert adaln_input is not Nonescale_msa, gate_msa, scale_mlp, gate_mlp = self.adaLN_modulation(adaln_input).chunk(4, dim=1)- x = x + gate_msa.unsqueeze(1).tanh() * self.attention_norm2(+ x = x + apply_gate(gate_msa.unsqueeze(1).tanh(), self.attention_norm2(clamp_fp16(self.attention(- modulate(self.attention_norm1(x), scale_msa),+ modulate(self.attention_norm1(x), scale_msa, timestep_zero_index=timestep_zero_index),x_mask,freqs_cis,transformer_options=transformer_options,- ))+ ))), timestep_zero_index=timestep_zero_index)- x = x + gate_mlp.unsqueeze(1).tanh() * self.ffn_norm2(+ x = x + apply_gate(gate_mlp.unsqueeze(1).tanh(), self.ffn_norm2(clamp_fp16(self.feed_forward(- modulate(self.ffn_norm1(x), scale_mlp),- ))+ modulate(self.ffn_norm1(x), scale_mlp, timestep_zero_index=timestep_zero_index),+ ))), timestep_zero_index=timestep_zero_index)else:assert adaln_input is None@@ -345,13 +389,37 @@ class FinalLayer(nn.Module):),)- def forward(self, x, c):+ def forward(self, x, c, timestep_zero_index=None):scale = self.adaLN_modulation(c)- x = modulate(self.norm_final(x), scale)+ x = modulate(self.norm_final(x), scale, timestep_zero_index=timestep_zero_index)x = self.linear(x)return x+def pad_zimage(feats, pad_token, pad_tokens_multiple):+ pad_extra = (-feats.shape[1]) % pad_tokens_multiple+ return torch.cat((feats, pad_token.to(device=feats.device, dtype=feats.dtype, copy=True).unsqueeze(0).repeat(feats.shape[0], pad_extra, 1)), dim=1), pad_extra+++def pos_ids_x(start_t, H_tokens, W_tokens, batch_size, device, transformer_options={}):+ rope_options = transformer_options.get("rope_options", None)+ h_scale = 1.0+ w_scale = 1.0+ h_start = 0+ w_start = 0+ if rope_options is not None:+ h_scale = rope_options.get("scale_y", 1.0)+ w_scale = rope_options.get("scale_x", 1.0)++ h_start = rope_options.get("shift_y", 0.0)+ w_start = rope_options.get("shift_x", 0.0)+ x_pos_ids = torch.zeros((batch_size, H_tokens * W_tokens, 3), dtype=torch.float32, device=device)+ x_pos_ids[:, :, 0] = start_t+ x_pos_ids[:, :, 1] = (torch.arange(H_tokens, dtype=torch.float32, device=device) * h_scale + h_start).view(-1, 1).repeat(1, W_tokens).flatten()+ x_pos_ids[:, :, 2] = (torch.arange(W_tokens, dtype=torch.float32, device=device) * w_scale + w_start).view(1, -1).repeat(H_tokens, 1).flatten()+ return x_pos_ids++class NextDiT(nn.Module):"""Diffusion model with a Transformer backbone.@@ -378,6 +446,7 @@ class NextDiT(nn.Module):time_scale=1.0,pad_tokens_multiple=None,clip_text_dim=None,+ siglip_feat_dim=None,image_model=None,device=None,dtype=None,@@ -491,6 +560,41 @@ class NextDiT(nn.Module):for layer_id in range(n_layers)])++ if siglip_feat_dim is not None:+ self.siglip_embedder = nn.Sequential(+ operation_settings.get("operations").RMSNorm(siglip_feat_dim, eps=norm_eps, elementwise_affine=True, device=operation_settings.get("device"), dtype=operation_settings.get("dtype")),+ operation_settings.get("operations").Linear(+ siglip_feat_dim,+ dim,+ bias=True,+ device=operation_settings.get("device"),+ dtype=operation_settings.get("dtype"),+ ),+ )+ self.siglip_refiner = nn.ModuleList(+ [+ JointTransformerBlock(+ layer_id,+ dim,+ n_heads,+ n_kv_heads,+ multiple_of,+ ffn_dim_multiplier,+ norm_eps,+ qk_norm,+ modulation=False,+ operation_settings=operation_settings,+ )+ for layer_id in range(n_refiner_layers)+ ]+ )+ self.siglip_pad_token = nn.Parameter(torch.empty((1, dim), device=device, dtype=dtype))+ else:+ self.siglip_embedder = None+ self.siglip_refiner = None+ self.siglip_pad_token = None+# This norm final is in the lumina 2.0 code but isn't actually used for anything.# self.norm_final = operation_settings.get("operations").RMSNorm(dim, eps=norm_eps, elementwise_affine=True, device=operation_settings.get("device"), dtype=operation_settings.get("dtype"))self.final_layer = FinalLayer(dim, patch_size, self.out_channels, z_image_modulation=z_image_modulation, operation_settings=operation_settings)@@ -531,70 +635,166 @@ class NextDiT(nn.Module):imgs = torch.stack(imgs, dim=0)return imgs- def patchify_and_embed(- self, x: List[torch.Tensor] | torch.Tensor, cap_feats: torch.Tensor, cap_mask: torch.Tensor, t: torch.Tensor, num_tokens, transformer_options={}- ) -> Tuple[torch.Tensor, torch.Tensor, List[Tuple[int, int]], List[int], torch.Tensor]:- bsz = len(x)+ def embed_cap(self, cap_feats=None, offset=0, bsz=1, device=None, dtype=None):+ if cap_feats is not None:+ cap_feats = self.cap_embedder(cap_feats)+ cap_feats_len = cap_feats.shape[1]+ if self.pad_tokens_multiple is not None:+ cap_feats, _ = pad_zimage(cap_feats, self.cap_pad_token, self.pad_tokens_multiple)+ else:+ cap_feats_len = 0+ cap_feats = self.cap_pad_token.to(device=device, dtype=dtype, copy=True).unsqueeze(0).repeat(bsz, self.pad_tokens_multiple, 1)++ cap_pos_ids = torch.zeros(bsz, cap_feats.shape[1], 3, dtype=torch.float32, device=device)+ cap_pos_ids[:, :, 0] = torch.arange(cap_feats.shape[1], dtype=torch.float32, device=device) + 1.0 + offset+ embeds = (cap_feats,)+ freqs_cis = (self.rope_embedder(cap_pos_ids).movedim(1, 2),)+ return embeds, freqs_cis, cap_feats_len++ def embed_all(self, x, cap_feats=None, siglip_feats=None, offset=0, omni=False, transformer_options={}):+ bsz = 1pH = pW = self.patch_size- device = x[0].device- orig_x = x+ device = x.device+ embeds, freqs_cis, cap_feats_len = self.embed_cap(cap_feats, offset=offset, bsz=bsz, device=device, dtype=x.dtype)- if self.pad_tokens_multiple is not None:- pad_extra = (-cap_feats.shape[1]) % self.pad_tokens_multiple- cap_feats = torch.cat((cap_feats, self.cap_pad_token.to(device=cap_feats.device, dtype=cap_feats.dtype, copy=True).unsqueeze(0).repeat(cap_feats.shape[0], pad_extra, 1)), dim=1)+ if not omni:+ cap_feats_len = embeds[0].shape[1] + offset+ embeds += (None,)+ freqs_cis += (None,)+ else:+ cap_feats_len += offset+ if siglip_feats is not None:+ b, h, w, c = siglip_feats.shape+ siglip_feats = siglip_feats.permute(0, 3, 1, 2).reshape(b, h * w, c)+ siglip_feats = self.siglip_embedder(siglip_feats)+ siglip_pos_ids = torch.zeros((bsz, siglip_feats.shape[1], 3), dtype=torch.float32, device=device)+ siglip_pos_ids[:, :, 0] = cap_feats_len + 2+ siglip_pos_ids[:, :, 1] = (torch.linspace(0, h * 8 - 1, steps=h, dtype=torch.float32, device=device).floor()).view(-1, 1).repeat(1, w).flatten()+ siglip_pos_ids[:, :, 2] = (torch.linspace(0, w * 8 - 1, steps=w, dtype=torch.float32, device=device).floor()).view(1, -1).repeat(h, 1).flatten()+ if self.siglip_pad_token is not None:+ siglip_feats, pad_extra = pad_zimage(siglip_feats, self.siglip_pad_token, self.pad_tokens_multiple) # TODO: double check+ siglip_pos_ids = torch.nn.functional.pad(siglip_pos_ids, (0, 0, 0, pad_extra))+ else:+ siglip_feats = self.siglip_pad_token.to(device=device, dtype=x.dtype, copy=True).unsqueeze(0).repeat(bsz, self.pad_tokens_multiple, 1)+ siglip_pos_ids = torch.zeros((bsz, siglip_feats.shape[1], 3), dtype=torch.float32, device=device)- cap_pos_ids = torch.zeros(bsz, cap_feats.shape[1], 3, dtype=torch.float32, device=device)- cap_pos_ids[:, :, 0] = torch.arange(cap_feats.shape[1], dtype=torch.float32, device=device) + 1.0+ if siglip_feats is None:+ embeds += (None,)+ freqs_cis += (None,)+ else:+ embeds += (siglip_feats,)+ freqs_cis += (self.rope_embedder(siglip_pos_ids).movedim(1, 2),)B, C, H, W = x.shapex = self.x_embedder(x.view(B, C, H // pH, pH, W // pW, pW).permute(0, 2, 4, 3, 5, 1).flatten(3).flatten(1, 2))+ x_pos_ids = pos_ids_x(cap_feats_len + 1, H // pH, W // pW, bsz, device, transformer_options=transformer_options)+ if self.pad_tokens_multiple is not None:+ x, pad_extra = pad_zimage(x, self.x_pad_token, self.pad_tokens_multiple)+ x_pos_ids = torch.nn.functional.pad(x_pos_ids, (0, 0, 0, pad_extra))- rope_options = transformer_options.get("rope_options", None)- h_scale = 1.0- w_scale = 1.0- h_start = 0- w_start = 0- if rope_options is not None:- h_scale = rope_options.get("scale_y", 1.0)- w_scale = rope_options.get("scale_x", 1.0)-- h_start = rope_options.get("shift_y", 0.0)- w_start = rope_options.get("shift_x", 0.0)+ embeds += (x,)+ freqs_cis += (self.rope_embedder(x_pos_ids).movedim(1, 2),)+ return embeds, freqs_cis, cap_feats_len + len(freqs_cis) - 1- H_tokens, W_tokens = H // pH, W // pW- x_pos_ids = torch.zeros((bsz, x.shape[1], 3), dtype=torch.float32, device=device)- x_pos_ids[:, :, 0] = cap_feats.shape[1] + 1- x_pos_ids[:, :, 1] = (torch.arange(H_tokens, dtype=torch.float32, device=device) * h_scale + h_start).view(-1, 1).repeat(1, W_tokens).flatten()- x_pos_ids[:, :, 2] = (torch.arange(W_tokens, dtype=torch.float32, device=device) * w_scale + w_start).view(1, -1).repeat(H_tokens, 1).flatten()- if self.pad_tokens_multiple is not None:- pad_extra = (-x.shape[1]) % self.pad_tokens_multiple- x = torch.cat((x, self.x_pad_token.to(device=x.device, dtype=x.dtype, copy=True).unsqueeze(0).repeat(x.shape[0], pad_extra, 1)), dim=1)- x_pos_ids = torch.nn.functional.pad(x_pos_ids, (0, 0, 0, pad_extra))+ def patchify_and_embed(+ self, x: torch.Tensor, cap_feats: torch.Tensor, cap_mask: torch.Tensor, t: torch.Tensor, num_tokens, ref_latents=[], ref_contexts=[], siglip_feats=[], transformer_options={}+ ) -> Tuple[torch.Tensor, torch.Tensor, List[Tuple[int, int]], List[int], torch.Tensor]:+ bsz = x.shape[0]+ cap_mask = None # TODO?+ main_siglip = None+ orig_x = x- freqs_cis = self.rope_embedder(torch.cat((cap_pos_ids, x_pos_ids), dim=1)).movedim(1, 2)+ embeds = ([], [], [])+ freqs_cis = ([], [], [])+ leftover_cap = []++ start_t = 0+ omni = len(ref_latents) > 0+ if omni:+ for i, ref in enumerate(ref_latents):+ if i < len(ref_contexts):+ ref_con = ref_contexts[i]+ else:+ ref_con = None+ if i < len(siglip_feats):+ sig_feat = siglip_feats[i]+ else:+ sig_feat = None++ out = self.embed_all(ref, ref_con, sig_feat, offset=start_t, omni=omni, transformer_options=transformer_options)+ for i, e in enumerate(out[0]):+ embeds[i].append(comfy.utils.repeat_to_batch_size(e, bsz))+ freqs_cis[i].append(out[1][i])+ start_t = out[2]+ leftover_cap = ref_contexts[len(ref_latents):]++ H, W = x.shape[-2], x.shape[-1]+ img_sizes = [(H, W)] * bsz+ out = self.embed_all(x, cap_feats, main_siglip, offset=start_t, omni=omni, transformer_options=transformer_options)+ img_len = out[0][-1].shape[1]+ cap_len = out[0][0].shape[1]+ for i, e in enumerate(out[0]):+ if e is not None:+ e = comfy.utils.repeat_to_batch_size(e, bsz)+ embeds[i].append(e)+ freqs_cis[i].append(out[1][i])+ start_t = out[2]++ for cap in leftover_cap:+ out = self.embed_cap(cap, offset=start_t, bsz=bsz, device=x.device, dtype=x.dtype)+ cap_len += out[0][0].shape[1]+ embeds[0].append(comfy.utils.repeat_to_batch_size(out[0][0], bsz))+ freqs_cis[0].append(out[1][0])+ start_t += out[2]patches = transformer_options.get("patches", {})# refine context+ cap_feats = torch.cat(embeds[0], dim=1)+ cap_freqs_cis = torch.cat(freqs_cis[0], dim=1)for layer in self.context_refiner:- cap_feats = layer(cap_feats, cap_mask, freqs_cis[:, :cap_pos_ids.shape[1]], transformer_options=transformer_options)+ cap_feats = layer(cap_feats, cap_mask, cap_freqs_cis, transformer_options=transformer_options)++ feats = (cap_feats,)+ fc = (cap_freqs_cis,)++ if omni:+ siglip_mask = None+ siglip_feats_combined = torch.cat(embeds[1], dim=1)+ siglip_feats_freqs_cis = torch.cat(freqs_cis[1], dim=1)+ if self.siglip_refiner is not None:+ for layer in self.siglip_refiner:+ siglip_feats_combined = layer(siglip_feats_combined, siglip_mask, siglip_feats_freqs_cis, transformer_options=transformer_options)+ feats += (siglip_feats_combined,)+ fc += (siglip_feats_freqs_cis,)padded_img_mask = None+ x = torch.cat(embeds[-1], dim=1)+ fc_x = torch.cat(freqs_cis[-1], dim=1)+ if omni:+ timestep_zero_index = [(x.shape[1] - img_len, x.shape[1])]+ else:+ timestep_zero_index = None+x_input = xfor i, layer in enumerate(self.noise_refiner):- x = layer(x, padded_img_mask, freqs_cis[:, cap_pos_ids.shape[1]:], t, transformer_options=transformer_options)+ x = layer(x, padded_img_mask, fc_x, t, timestep_zero_index=timestep_zero_index, transformer_options=transformer_options)if "noise_refiner" in patches:for p in patches["noise_refiner"]:- out = p({"img": x, "img_input": x_input, "txt": cap_feats, "pe": freqs_cis[:, cap_pos_ids.shape[1]:], "vec": t, "x": orig_x, "block_index": i, "transformer_options": transformer_options, "block_type": "noise_refiner"})+ out = p({"img": x, "img_input": x_input, "txt": cap_feats, "pe": fc_x, "vec": t, "x": orig_x, "block_index": i, "transformer_options": transformer_options, "block_type": "noise_refiner"})if "img" in out:x = out["img"]- padded_full_embed = torch.cat((cap_feats, x), dim=1)+ padded_full_embed = torch.cat(feats + (x,), dim=1)+ if timestep_zero_index is not None:+ ind = padded_full_embed.shape[1] - x.shape[1]+ timestep_zero_index = [(ind + x.shape[1] - img_len, ind + x.shape[1])]+ timestep_zero_index.append((feats[0].shape[1] - cap_len, feats[0].shape[1]))+mask = None- img_sizes = [(H, W)] * bsz- l_effective_cap_len = [cap_feats.shape[1]] * bsz- return padded_full_embed, mask, img_sizes, l_effective_cap_len, freqs_cis+ l_effective_cap_len = [padded_full_embed.shape[1] - img_len] * bsz+ return padded_full_embed, mask, img_sizes, l_effective_cap_len, torch.cat(fc + (fc_x,), dim=1), timestep_zero_indexdef forward(self, x, timesteps, context, num_tokens, attention_mask=None, **kwargs):return comfy.patcher_extension.WrapperExecutor.new_class_executor(@@ -604,7 +804,11 @@ class NextDiT(nn.Module):).execute(x, timesteps, context, num_tokens, attention_mask, **kwargs)# def forward(self, x, t, cap_feats, cap_mask):- def _forward(self, x, timesteps, context, num_tokens, attention_mask=None, transformer_options={}, **kwargs):+ def _forward(self, x, timesteps, context, num_tokens, attention_mask=None, ref_latents=[], ref_contexts=[], siglip_feats=[], transformer_options={}, **kwargs):+ omni = len(ref_latents) > 0+ if omni:+ timesteps = torch.cat([timesteps * 0, timesteps], dim=0)+t = 1.0 - timestepscap_feats = contextcap_mask = attention_mask@@ -619,8 +823,6 @@ class NextDiT(nn.Module):t = self.t_embedder(t * self.time_scale, dtype=x.dtype) # (N, D)adaln_input = t- cap_feats = self.cap_embedder(cap_feats) # (N, L, D) # todo check if able to batchify w.o. redundant compute-if self.clip_text_pooled_proj is not None:pooled = kwargs.get("clip_text_pooled", None)if pooled is not None:@@ -632,7 +834,7 @@ class NextDiT(nn.Module):patches = transformer_options.get("patches", {})x_is_tensor = isinstance(x, torch.Tensor)- img, mask, img_size, cap_size, freqs_cis = self.patchify_and_embed(x, cap_feats, cap_mask, adaln_input, num_tokens, transformer_options=transformer_options)+ img, mask, img_size, cap_size, freqs_cis, timestep_zero_index = self.patchify_and_embed(x, cap_feats, cap_mask, adaln_input, num_tokens, ref_latents=ref_latents, ref_contexts=ref_contexts, siglip_feats=siglip_feats, transformer_options=transformer_options)freqs_cis = freqs_cis.to(img.device)transformer_options["total_blocks"] = len(self.layers)@@ -640,7 +842,7 @@ class NextDiT(nn.Module):img_input = imgfor i, layer in enumerate(self.layers):transformer_options["block_index"] = i- img = layer(img, mask, freqs_cis, adaln_input, transformer_options=transformer_options)+ img = layer(img, mask, freqs_cis, adaln_input, timestep_zero_index=timestep_zero_index, transformer_options=transformer_options)if "double_block" in patches:for p in patches["double_block"]:out = p({"img": img[:, cap_size[0]:], "img_input": img_input[:, cap_size[0]:], "txt": img[:, :cap_size[0]], "pe": freqs_cis[:, cap_size[0]:], "vec": adaln_input, "x": x, "block_index": i, "transformer_options": transformer_options})@@ -649,8 +851,7 @@ class NextDiT(nn.Module):if "txt" in out:img[:, :cap_size[0]] = out["txt"]- img = self.final_layer(img, adaln_input)+ img = self.final_layer(img, adaln_input, timestep_zero_index=timestep_zero_index)img = self.unpatchify(img, img_size, cap_size, return_tensor=x_is_tensor)[:, :, :h, :w]-return -imgdiff --git a/comfy/model_base.py b/comfy/model_base.pyindex 49efd700b..28ba2643e 100644--- a/comfy/model_base.py+++ b/comfy/model_base.py@@ -1150,6 +1150,7 @@ class CosmosPredict2(BaseModel):class Lumina2(BaseModel):def __init__(self, model_config, model_type=ModelType.FLOW, device=None):super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.lumina.model.NextDiT)+ self.memory_usage_factor_conds = ("ref_latents",)def extra_conds(self, **kwargs):out = super().extra_conds(**kwargs)@@ -1169,6 +1170,35 @@ class Lumina2(BaseModel):if clip_text_pooled is not None:out['clip_text_pooled'] = comfy.conds.CONDRegular(clip_text_pooled)+ clip_vision_outputs = kwargs.get("clip_vision_outputs", list(map(lambda a: a.get("clip_vision_output"), kwargs.get("unclip_conditioning", [{}])))) # Z Image omni+ if clip_vision_outputs is not None and len(clip_vision_outputs) > 0:+ sigfeats = []+ for clip_vision_output in clip_vision_outputs:+ if clip_vision_output is not None:+ image_size = clip_vision_output.image_sizes[0]+ shape = clip_vision_output.last_hidden_state.shape+ sigfeats.append(clip_vision_output.last_hidden_state.reshape(shape[0], image_size[1] // 16, image_size[2] // 16, shape[-1]))+ if len(sigfeats) > 0:+ out['siglip_feats'] = comfy.conds.CONDList(sigfeats)++ ref_latents = kwargs.get("reference_latents", None)+ if ref_latents is not None:+ latents = []+ for lat in ref_latents:+ latents.append(self.process_latent_in(lat))+ out['ref_latents'] = comfy.conds.CONDList(latents)++ ref_contexts = kwargs.get("reference_latents_text_embeds", None)+ if ref_contexts is not None:+ out['ref_contexts'] = comfy.conds.CONDList(ref_contexts)++ return out++ def extra_conds_shapes(self, **kwargs):+ out = {}+ ref_latents = kwargs.get("reference_latents", None)+ if ref_latents is not None:+ out['ref_latents'] = list([1, 16, sum(map(lambda a: math.prod(a.size()[2:]), ref_latents))])return outclass WAN21(BaseModel):diff --git a/comfy/model_detection.py b/comfy/model_detection.pyindex aff5a50b9..42884f797 100644--- a/comfy/model_detection.py+++ b/comfy/model_detection.py@@ -446,6 +446,9 @@ def detect_unet_config(state_dict, key_prefix, metadata=None):dit_config["time_scale"] = 1000.0if '{}cap_pad_token'.format(key_prefix) in state_dict_keys:dit_config["pad_tokens_multiple"] = 32+ sig_weight = state_dict.get('{}siglip_embedder.0.weight'.format(key_prefix), None)+ if sig_weight is not None:+ dit_config["siglip_feat_dim"] = sig_weight.shape[0]return dit_configdiff --git a/comfy_extras/nodes_zimage.py b/comfy_extras/nodes_zimage.pynew file mode 100644index 000000000..2ee3c43b1--- /dev/null+++ b/comfy_extras/nodes_zimage.py@@ -0,0 +1,88 @@+import node_helpers+from typing_extensions import override+from comfy_api.latest import ComfyExtension, io+import math+import comfy.utils+++class TextEncodeZImageOmni(io.ComfyNode):+ @classmethod+ def define_schema(cls):+ return io.Schema(+ node_id="TextEncodeZImageOmni",+ category="advanced/conditioning",+ is_experimental=True,+ inputs=[+ io.Clip.Input("clip"),+ io.ClipVision.Input("image_encoder", optional=True),+ io.String.Input("prompt", multiline=True, dynamic_prompts=True),+ io.Boolean.Input("auto_resize_images", default=True),+ io.Vae.Input("vae", optional=True),+ io.Image.Input("image1", optional=True),+ io.Image.Input("image2", optional=True),+ io.Image.Input("image3", optional=True),+ ],+ outputs=[+ io.Conditioning.Output(),+ ],+ )++ @classmethod+ def execute(cls, clip, prompt, image_encoder=None, auto_resize_images=True, vae=None, image1=None, image2=None, image3=None) -> io.NodeOutput:+ ref_latents = []+ images = list(filter(lambda a: a is not None, [image1, image2, image3]))++ prompt_list = []+ template = None+ if len(images) > 0:+ prompt_list = ["<|im_start|>user\n<|vision_start|>"]+ prompt_list += ["<|vision_end|><|vision_start|>"] * (len(images) - 1)+ prompt_list += ["<|vision_end|><|im_end|>"]+ template = "<|vision_end|>{}<|im_end|>\n<|im_start|>assistant\n<|vision_start|>"++ encoded_images = []++ for i, image in enumerate(images):+ if image_encoder is not None:+ encoded_images.append(image_encoder.encode_image(image))++ if vae is not None:+ if auto_resize_images:+ samples = image.movedim(-1, 1)+ total = int(1024 * 1024)+ scale_by = math.sqrt(total / (samples.shape[3] * samples.shape[2]))+ width = round(samples.shape[3] * scale_by / 8.0) * 8+ height = round(samples.shape[2] * scale_by / 8.0) * 8++ image = comfy.utils.common_upscale(samples, width, height, "area", "disabled").movedim(1, -1)+ ref_latents.append(vae.encode(image))++ tokens = clip.tokenize(prompt, llama_template=template)+ conditioning = clip.encode_from_tokens_scheduled(tokens)++ extra_text_embeds = []+ for p in prompt_list:+ tokens = clip.tokenize(p, llama_template="{}")+ text_embeds = clip.encode_from_tokens_scheduled(tokens)+ extra_text_embeds.append(text_embeds[0][0])++ if len(ref_latents) > 0:+ conditioning = node_helpers.conditioning_set_values(conditioning, {"reference_latents": ref_latents}, append=True)+ if len(encoded_images) > 0:+ conditioning = node_helpers.conditioning_set_values(conditioning, {"clip_vision_outputs": encoded_images}, append=True)+ if len(extra_text_embeds) > 0:+ conditioning = node_helpers.conditioning_set_values(conditioning, {"reference_latents_text_embeds": extra_text_embeds}, append=True)++ return io.NodeOutput(conditioning)+++class ZImageExtension(ComfyExtension):+ @override+ async def get_node_list(self) -> list[type[io.ComfyNode]]:+ return [+ TextEncodeZImageOmni,+ ]+++async def comfy_entrypoint() -> ZImageExtension:+ return ZImageExtension()diff --git a/nodes.py b/nodes.pyindex cba8eacc2..ea5d6e525 100644--- a/nodes.py+++ b/nodes.py@@ -2373,6 +2373,7 @@ async def init_builtin_extra_nodes():"nodes_kandinsky5.py","nodes_wanmove.py","nodes_image_compare.py",+ "nodes_zimage.py",]import_failed = []