Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Support zimage omni base model. (#11979)

comfyanonymous2108167f9f70
diff --git a/comfy/ldm/lumina/model.py b/comfy/ldm/lumina/model.py
index afbab2ac7..139f879a1 100644
--- a/comfy/ldm/lumina/model.py
+++ b/comfy/ldm/lumina/model.py
@@ -13,10 +13,53 @@ from comfy.ldm.modules.attention import optimized_attention_masked
from comfy.ldm.flux.layers import EmbedND
from comfy.ldm.flux.math import apply_rope
import comfy.patcher_extension
+import comfy.utils
-def modulate(x, scale):
- return x * (1 + scale.unsqueeze(1))
+def invert_slices(slices, length):
+ sorted_slices = sorted(slices)
+ result = []
+ current = 0
+
+ for start, end in sorted_slices:
+ if current < start:
+ result.append((current, start))
+ current = max(current, end)
+
+ if current < length:
+ result.append((current, length))
+
+ return result
+
+
+def modulate(x, scale, timestep_zero_index=None):
+ if timestep_zero_index is None:
+ return x * (1 + scale.unsqueeze(1))
+ else:
+ scale = (1 + scale.unsqueeze(1))
+ actual_batch = scale.size(0) // 2
+ slices = timestep_zero_index
+ invert = invert_slices(timestep_zero_index, x.shape[1])
+ for s in slices:
+ x[:, s[0]:s[1]] *= scale[actual_batch:]
+ for s in invert:
+ x[:, s[0]:s[1]] *= scale[:actual_batch]
+ return x
+
+
+def apply_gate(gate, x, timestep_zero_index=None):
+ if timestep_zero_index is None:
+ return gate * x
+ else:
+ actual_batch = gate.size(0) // 2
+
+ slices = timestep_zero_index
+ invert = invert_slices(timestep_zero_index, x.shape[1])
+ for s in slices:
+ x[:, s[0]:s[1]] *= gate[actual_batch:]
+ for s in invert:
+ x[:, s[0]:s[1]] *= gate[:actual_batch]
+ return x
#############################################################################
# Core NextDiT Model #
@@ -258,6 +301,7 @@ class JointTransformerBlock(nn.Module):
x_mask: torch.Tensor,
freqs_cis: torch.Tensor,
adaln_input: Optional[torch.Tensor]=None,
+ timestep_zero_index=None,
transformer_options={},
):
"""
@@ -276,18 +320,18 @@ class JointTransformerBlock(nn.Module):
assert adaln_input is not None
scale_msa, gate_msa, scale_mlp, gate_mlp = self.adaLN_modulation(adaln_input).chunk(4, dim=1)
- x = x + gate_msa.unsqueeze(1).tanh() * self.attention_norm2(
+ x = x + apply_gate(gate_msa.unsqueeze(1).tanh(), self.attention_norm2(
clamp_fp16(self.attention(
- modulate(self.attention_norm1(x), scale_msa),
+ modulate(self.attention_norm1(x), scale_msa, timestep_zero_index=timestep_zero_index),
x_mask,
freqs_cis,
transformer_options=transformer_options,
- ))
+ ))), timestep_zero_index=timestep_zero_index
)
- x = x + gate_mlp.unsqueeze(1).tanh() * self.ffn_norm2(
+ x = x + apply_gate(gate_mlp.unsqueeze(1).tanh(), self.ffn_norm2(
clamp_fp16(self.feed_forward(
- modulate(self.ffn_norm1(x), scale_mlp),
- ))
+ modulate(self.ffn_norm1(x), scale_mlp, timestep_zero_index=timestep_zero_index),
+ ))), timestep_zero_index=timestep_zero_index
)
else:
assert adaln_input is None
@@ -345,13 +389,37 @@ class FinalLayer(nn.Module):
),
)
- def forward(self, x, c):
+ def forward(self, x, c, timestep_zero_index=None):
scale = self.adaLN_modulation(c)
- x = modulate(self.norm_final(x), scale)
+ x = modulate(self.norm_final(x), scale, timestep_zero_index=timestep_zero_index)
x = self.linear(x)
return x
+def pad_zimage(feats, pad_token, pad_tokens_multiple):
+ pad_extra = (-feats.shape[1]) % pad_tokens_multiple
+ return torch.cat((feats, pad_token.to(device=feats.device, dtype=feats.dtype, copy=True).unsqueeze(0).repeat(feats.shape[0], pad_extra, 1)), dim=1), pad_extra
+
+
+def pos_ids_x(start_t, H_tokens, W_tokens, batch_size, device, transformer_options={}):
+ rope_options = transformer_options.get("rope_options", None)
+ h_scale = 1.0
+ w_scale = 1.0
+ h_start = 0
+ w_start = 0
+ if rope_options is not None:
+ h_scale = rope_options.get("scale_y", 1.0)
+ w_scale = rope_options.get("scale_x", 1.0)
+
+ h_start = rope_options.get("shift_y", 0.0)
+ w_start = rope_options.get("shift_x", 0.0)
+ x_pos_ids = torch.zeros((batch_size, H_tokens * W_tokens, 3), dtype=torch.float32, device=device)
+ x_pos_ids[:, :, 0] = start_t
+ x_pos_ids[:, :, 1] = (torch.arange(H_tokens, dtype=torch.float32, device=device) * h_scale + h_start).view(-1, 1).repeat(1, W_tokens).flatten()
+ x_pos_ids[:, :, 2] = (torch.arange(W_tokens, dtype=torch.float32, device=device) * w_scale + w_start).view(1, -1).repeat(H_tokens, 1).flatten()
+ return x_pos_ids
+
+
class NextDiT(nn.Module):
"""
Diffusion model with a Transformer backbone.
@@ -378,6 +446,7 @@ class NextDiT(nn.Module):
time_scale=1.0,
pad_tokens_multiple=None,
clip_text_dim=None,
+ siglip_feat_dim=None,
image_model=None,
device=None,
dtype=None,
@@ -491,6 +560,41 @@ class NextDiT(nn.Module):
for layer_id in range(n_layers)
]
)
+
+ if siglip_feat_dim is not None:
+ self.siglip_embedder = nn.Sequential(
+ operation_settings.get("operations").RMSNorm(siglip_feat_dim, eps=norm_eps, elementwise_affine=True, device=operation_settings.get("device"), dtype=operation_settings.get("dtype")),
+ operation_settings.get("operations").Linear(
+ siglip_feat_dim,
+ dim,
+ bias=True,
+ device=operation_settings.get("device"),
+ dtype=operation_settings.get("dtype"),
+ ),
+ )
+ self.siglip_refiner = nn.ModuleList(
+ [
+ JointTransformerBlock(
+ layer_id,
+ dim,
+ n_heads,
+ n_kv_heads,
+ multiple_of,
+ ffn_dim_multiplier,
+ norm_eps,
+ qk_norm,
+ modulation=False,
+ operation_settings=operation_settings,
+ )
+ for layer_id in range(n_refiner_layers)
+ ]
+ )
+ self.siglip_pad_token = nn.Parameter(torch.empty((1, dim), device=device, dtype=dtype))
+ else:
+ self.siglip_embedder = None
+ self.siglip_refiner = None
+ self.siglip_pad_token = None
+
# This norm final is in the lumina 2.0 code but isn't actually used for anything.
# self.norm_final = operation_settings.get("operations").RMSNorm(dim, eps=norm_eps, elementwise_affine=True, device=operation_settings.get("device"), dtype=operation_settings.get("dtype"))
self.final_layer = FinalLayer(dim, patch_size, self.out_channels, z_image_modulation=z_image_modulation, operation_settings=operation_settings)
@@ -531,70 +635,166 @@ class NextDiT(nn.Module):
imgs = torch.stack(imgs, dim=0)
return imgs
- def patchify_and_embed(
- self, x: List[torch.Tensor] | torch.Tensor, cap_feats: torch.Tensor, cap_mask: torch.Tensor, t: torch.Tensor, num_tokens, transformer_options={}
- ) -> Tuple[torch.Tensor, torch.Tensor, List[Tuple[int, int]], List[int], torch.Tensor]:
- bsz = len(x)
+ def embed_cap(self, cap_feats=None, offset=0, bsz=1, device=None, dtype=None):
+ if cap_feats is not None:
+ cap_feats = self.cap_embedder(cap_feats)
+ cap_feats_len = cap_feats.shape[1]
+ if self.pad_tokens_multiple is not None:
+ cap_feats, _ = pad_zimage(cap_feats, self.cap_pad_token, self.pad_tokens_multiple)
+ else:
+ cap_feats_len = 0
+ cap_feats = self.cap_pad_token.to(device=device, dtype=dtype, copy=True).unsqueeze(0).repeat(bsz, self.pad_tokens_multiple, 1)
+
+ cap_pos_ids = torch.zeros(bsz, cap_feats.shape[1], 3, dtype=torch.float32, device=device)
+ cap_pos_ids[:, :, 0] = torch.arange(cap_feats.shape[1], dtype=torch.float32, device=device) + 1.0 + offset
+ embeds = (cap_feats,)
+ freqs_cis = (self.rope_embedder(cap_pos_ids).movedim(1, 2),)
+ return embeds, freqs_cis, cap_feats_len
+
+ def embed_all(self, x, cap_feats=None, siglip_feats=None, offset=0, omni=False, transformer_options={}):
+ bsz = 1
pH = pW = self.patch_size
- device = x[0].device
- orig_x = x
+ device = x.device
+ embeds, freqs_cis, cap_feats_len = self.embed_cap(cap_feats, offset=offset, bsz=bsz, device=device, dtype=x.dtype)
- if self.pad_tokens_multiple is not None:
- pad_extra = (-cap_feats.shape[1]) % self.pad_tokens_multiple
- cap_feats = torch.cat((cap_feats, self.cap_pad_token.to(device=cap_feats.device, dtype=cap_feats.dtype, copy=True).unsqueeze(0).repeat(cap_feats.shape[0], pad_extra, 1)), dim=1)
+ if not omni:
+ cap_feats_len = embeds[0].shape[1] + offset
+ embeds += (None,)
+ freqs_cis += (None,)
+ else:
+ cap_feats_len += offset
+ if siglip_feats is not None:
+ b, h, w, c = siglip_feats.shape
+ siglip_feats = siglip_feats.permute(0, 3, 1, 2).reshape(b, h * w, c)
+ siglip_feats = self.siglip_embedder(siglip_feats)
+ siglip_pos_ids = torch.zeros((bsz, siglip_feats.shape[1], 3), dtype=torch.float32, device=device)
+ siglip_pos_ids[:, :, 0] = cap_feats_len + 2
+ siglip_pos_ids[:, :, 1] = (torch.linspace(0, h * 8 - 1, steps=h, dtype=torch.float32, device=device).floor()).view(-1, 1).repeat(1, w).flatten()
+ siglip_pos_ids[:, :, 2] = (torch.linspace(0, w * 8 - 1, steps=w, dtype=torch.float32, device=device).floor()).view(1, -1).repeat(h, 1).flatten()
+ if self.siglip_pad_token is not None:
+ siglip_feats, pad_extra = pad_zimage(siglip_feats, self.siglip_pad_token, self.pad_tokens_multiple) # TODO: double check
+ siglip_pos_ids = torch.nn.functional.pad(siglip_pos_ids, (0, 0, 0, pad_extra))
+ else:
+ siglip_feats = self.siglip_pad_token.to(device=device, dtype=x.dtype, copy=True).unsqueeze(0).repeat(bsz, self.pad_tokens_multiple, 1)
+ siglip_pos_ids = torch.zeros((bsz, siglip_feats.shape[1], 3), dtype=torch.float32, device=device)
- cap_pos_ids = torch.zeros(bsz, cap_feats.shape[1], 3, dtype=torch.float32, device=device)
- cap_pos_ids[:, :, 0] = torch.arange(cap_feats.shape[1], dtype=torch.float32, device=device) + 1.0
+ if siglip_feats is None:
+ embeds += (None,)
+ freqs_cis += (None,)
+ else:
+ embeds += (siglip_feats,)
+ freqs_cis += (self.rope_embedder(siglip_pos_ids).movedim(1, 2),)
B, C, H, W = x.shape
x = self.x_embedder(x.view(B, C, H // pH, pH, W // pW, pW).permute(0, 2, 4, 3, 5, 1).flatten(3).flatten(1, 2))
+ x_pos_ids = pos_ids_x(cap_feats_len + 1, H // pH, W // pW, bsz, device, transformer_options=transformer_options)
+ if self.pad_tokens_multiple is not None:
+ x, pad_extra = pad_zimage(x, self.x_pad_token, self.pad_tokens_multiple)
+ x_pos_ids = torch.nn.functional.pad(x_pos_ids, (0, 0, 0, pad_extra))
- rope_options = transformer_options.get("rope_options", None)
- h_scale = 1.0
- w_scale = 1.0
- h_start = 0
- w_start = 0
- if rope_options is not None:
- h_scale = rope_options.get("scale_y", 1.0)
- w_scale = rope_options.get("scale_x", 1.0)
-
- h_start = rope_options.get("shift_y", 0.0)
- w_start = rope_options.get("shift_x", 0.0)
+ embeds += (x,)
+ freqs_cis += (self.rope_embedder(x_pos_ids).movedim(1, 2),)
+ return embeds, freqs_cis, cap_feats_len + len(freqs_cis) - 1
- H_tokens, W_tokens = H // pH, W // pW
- x_pos_ids = torch.zeros((bsz, x.shape[1], 3), dtype=torch.float32, device=device)
- x_pos_ids[:, :, 0] = cap_feats.shape[1] + 1
- x_pos_ids[:, :, 1] = (torch.arange(H_tokens, dtype=torch.float32, device=device) * h_scale + h_start).view(-1, 1).repeat(1, W_tokens).flatten()
- x_pos_ids[:, :, 2] = (torch.arange(W_tokens, dtype=torch.float32, device=device) * w_scale + w_start).view(1, -1).repeat(H_tokens, 1).flatten()
- if self.pad_tokens_multiple is not None:
- pad_extra = (-x.shape[1]) % self.pad_tokens_multiple
- x = torch.cat((x, self.x_pad_token.to(device=x.device, dtype=x.dtype, copy=True).unsqueeze(0).repeat(x.shape[0], pad_extra, 1)), dim=1)
- x_pos_ids = torch.nn.functional.pad(x_pos_ids, (0, 0, 0, pad_extra))
+ def patchify_and_embed(
+ self, x: torch.Tensor, cap_feats: torch.Tensor, cap_mask: torch.Tensor, t: torch.Tensor, num_tokens, ref_latents=[], ref_contexts=[], siglip_feats=[], transformer_options={}
+ ) -> Tuple[torch.Tensor, torch.Tensor, List[Tuple[int, int]], List[int], torch.Tensor]:
+ bsz = x.shape[0]
+ cap_mask = None # TODO?
+ main_siglip = None
+ orig_x = x
- freqs_cis = self.rope_embedder(torch.cat((cap_pos_ids, x_pos_ids), dim=1)).movedim(1, 2)
+ embeds = ([], [], [])
+ freqs_cis = ([], [], [])
+ leftover_cap = []
+
+ start_t = 0
+ omni = len(ref_latents) > 0
+ if omni:
+ for i, ref in enumerate(ref_latents):
+ if i < len(ref_contexts):
+ ref_con = ref_contexts[i]
+ else:
+ ref_con = None
+ if i < len(siglip_feats):
+ sig_feat = siglip_feats[i]
+ else:
+ sig_feat = None
+
+ out = self.embed_all(ref, ref_con, sig_feat, offset=start_t, omni=omni, transformer_options=transformer_options)
+ for i, e in enumerate(out[0]):
+ embeds[i].append(comfy.utils.repeat_to_batch_size(e, bsz))
+ freqs_cis[i].append(out[1][i])
+ start_t = out[2]
+ leftover_cap = ref_contexts[len(ref_latents):]
+
+ H, W = x.shape[-2], x.shape[-1]
+ img_sizes = [(H, W)] * bsz
+ out = self.embed_all(x, cap_feats, main_siglip, offset=start_t, omni=omni, transformer_options=transformer_options)
+ img_len = out[0][-1].shape[1]
+ cap_len = out[0][0].shape[1]
+ for i, e in enumerate(out[0]):
+ if e is not None:
+ e = comfy.utils.repeat_to_batch_size(e, bsz)
+ embeds[i].append(e)
+ freqs_cis[i].append(out[1][i])
+ start_t = out[2]
+
+ for cap in leftover_cap:
+ out = self.embed_cap(cap, offset=start_t, bsz=bsz, device=x.device, dtype=x.dtype)
+ cap_len += out[0][0].shape[1]
+ embeds[0].append(comfy.utils.repeat_to_batch_size(out[0][0], bsz))
+ freqs_cis[0].append(out[1][0])
+ start_t += out[2]
patches = transformer_options.get("patches", {})
# refine context
+ cap_feats = torch.cat(embeds[0], dim=1)
+ cap_freqs_cis = torch.cat(freqs_cis[0], dim=1)
for layer in self.context_refiner:
- cap_feats = layer(cap_feats, cap_mask, freqs_cis[:, :cap_pos_ids.shape[1]], transformer_options=transformer_options)
+ cap_feats = layer(cap_feats, cap_mask, cap_freqs_cis, transformer_options=transformer_options)
+
+ feats = (cap_feats,)
+ fc = (cap_freqs_cis,)
+
+ if omni:
+ siglip_mask = None
+ siglip_feats_combined = torch.cat(embeds[1], dim=1)
+ siglip_feats_freqs_cis = torch.cat(freqs_cis[1], dim=1)
+ if self.siglip_refiner is not None:
+ for layer in self.siglip_refiner:
+ siglip_feats_combined = layer(siglip_feats_combined, siglip_mask, siglip_feats_freqs_cis, transformer_options=transformer_options)
+ feats += (siglip_feats_combined,)
+ fc += (siglip_feats_freqs_cis,)
padded_img_mask = None
+ x = torch.cat(embeds[-1], dim=1)
+ fc_x = torch.cat(freqs_cis[-1], dim=1)
+ if omni:
+ timestep_zero_index = [(x.shape[1] - img_len, x.shape[1])]
+ else:
+ timestep_zero_index = None
+
x_input = x
for i, layer in enumerate(self.noise_refiner):
- x = layer(x, padded_img_mask, freqs_cis[:, cap_pos_ids.shape[1]:], t, transformer_options=transformer_options)
+ x = layer(x, padded_img_mask, fc_x, t, timestep_zero_index=timestep_zero_index, transformer_options=transformer_options)
if "noise_refiner" in patches:
for p in patches["noise_refiner"]:
- out = p({"img": x, "img_input": x_input, "txt": cap_feats, "pe": freqs_cis[:, cap_pos_ids.shape[1]:], "vec": t, "x": orig_x, "block_index": i, "transformer_options": transformer_options, "block_type": "noise_refiner"})
+ out = p({"img": x, "img_input": x_input, "txt": cap_feats, "pe": fc_x, "vec": t, "x": orig_x, "block_index": i, "transformer_options": transformer_options, "block_type": "noise_refiner"})
if "img" in out:
x = out["img"]
- padded_full_embed = torch.cat((cap_feats, x), dim=1)
+ padded_full_embed = torch.cat(feats + (x,), dim=1)
+ if timestep_zero_index is not None:
+ ind = padded_full_embed.shape[1] - x.shape[1]
+ timestep_zero_index = [(ind + x.shape[1] - img_len, ind + x.shape[1])]
+ timestep_zero_index.append((feats[0].shape[1] - cap_len, feats[0].shape[1]))
+
mask = None
- img_sizes = [(H, W)] * bsz
- l_effective_cap_len = [cap_feats.shape[1]] * bsz
- return padded_full_embed, mask, img_sizes, l_effective_cap_len, freqs_cis
+ l_effective_cap_len = [padded_full_embed.shape[1] - img_len] * bsz
+ return padded_full_embed, mask, img_sizes, l_effective_cap_len, torch.cat(fc + (fc_x,), dim=1), timestep_zero_index
def forward(self, x, timesteps, context, num_tokens, attention_mask=None, **kwargs):
return comfy.patcher_extension.WrapperExecutor.new_class_executor(
@@ -604,7 +804,11 @@ class NextDiT(nn.Module):
).execute(x, timesteps, context, num_tokens, attention_mask, **kwargs)
# def forward(self, x, t, cap_feats, cap_mask):
- def _forward(self, x, timesteps, context, num_tokens, attention_mask=None, transformer_options={}, **kwargs):
+ def _forward(self, x, timesteps, context, num_tokens, attention_mask=None, ref_latents=[], ref_contexts=[], siglip_feats=[], transformer_options={}, **kwargs):
+ omni = len(ref_latents) > 0
+ if omni:
+ timesteps = torch.cat([timesteps * 0, timesteps], dim=0)
+
t = 1.0 - timesteps
cap_feats = context
cap_mask = attention_mask
@@ -619,8 +823,6 @@ class NextDiT(nn.Module):
t = self.t_embedder(t * self.time_scale, dtype=x.dtype) # (N, D)
adaln_input = t
- cap_feats = self.cap_embedder(cap_feats) # (N, L, D) # todo check if able to batchify w.o. redundant compute
-
if self.clip_text_pooled_proj is not None:
pooled = kwargs.get("clip_text_pooled", None)
if pooled is not None:
@@ -632,7 +834,7 @@ class NextDiT(nn.Module):
patches = transformer_options.get("patches", {})
x_is_tensor = isinstance(x, torch.Tensor)
- img, mask, img_size, cap_size, freqs_cis = self.patchify_and_embed(x, cap_feats, cap_mask, adaln_input, num_tokens, transformer_options=transformer_options)
+ img, mask, img_size, cap_size, freqs_cis, timestep_zero_index = self.patchify_and_embed(x, cap_feats, cap_mask, adaln_input, num_tokens, ref_latents=ref_latents, ref_contexts=ref_contexts, siglip_feats=siglip_feats, transformer_options=transformer_options)
freqs_cis = freqs_cis.to(img.device)
transformer_options["total_blocks"] = len(self.layers)
@@ -640,7 +842,7 @@ class NextDiT(nn.Module):
img_input = img
for i, layer in enumerate(self.layers):
transformer_options["block_index"] = i
- img = layer(img, mask, freqs_cis, adaln_input, transformer_options=transformer_options)
+ img = layer(img, mask, freqs_cis, adaln_input, timestep_zero_index=timestep_zero_index, transformer_options=transformer_options)
if "double_block" in patches:
for p in patches["double_block"]:
out = p({"img": img[:, cap_size[0]:], "img_input": img_input[:, cap_size[0]:], "txt": img[:, :cap_size[0]], "pe": freqs_cis[:, cap_size[0]:], "vec": adaln_input, "x": x, "block_index": i, "transformer_options": transformer_options})
@@ -649,8 +851,7 @@ class NextDiT(nn.Module):
if "txt" in out:
img[:, :cap_size[0]] = out["txt"]
- img = self.final_layer(img, adaln_input)
+ img = self.final_layer(img, adaln_input, timestep_zero_index=timestep_zero_index)
img = self.unpatchify(img, img_size, cap_size, return_tensor=x_is_tensor)[:, :, :h, :w]
-
return -img
diff --git a/comfy/model_base.py b/comfy/model_base.py
index 49efd700b..28ba2643e 100644
--- a/comfy/model_base.py
+++ b/comfy/model_base.py
@@ -1150,6 +1150,7 @@ class CosmosPredict2(BaseModel):
class Lumina2(BaseModel):
def __init__(self, model_config, model_type=ModelType.FLOW, device=None):
super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.lumina.model.NextDiT)
+ self.memory_usage_factor_conds = ("ref_latents",)
def extra_conds(self, **kwargs):
out = super().extra_conds(**kwargs)
@@ -1169,6 +1170,35 @@ class Lumina2(BaseModel):
if clip_text_pooled is not None:
out['clip_text_pooled'] = comfy.conds.CONDRegular(clip_text_pooled)
+ clip_vision_outputs = kwargs.get("clip_vision_outputs", list(map(lambda a: a.get("clip_vision_output"), kwargs.get("unclip_conditioning", [{}])))) # Z Image omni
+ if clip_vision_outputs is not None and len(clip_vision_outputs) > 0:
+ sigfeats = []
+ for clip_vision_output in clip_vision_outputs:
+ if clip_vision_output is not None:
+ image_size = clip_vision_output.image_sizes[0]
+ shape = clip_vision_output.last_hidden_state.shape
+ sigfeats.append(clip_vision_output.last_hidden_state.reshape(shape[0], image_size[1] // 16, image_size[2] // 16, shape[-1]))
+ if len(sigfeats) > 0:
+ out['siglip_feats'] = comfy.conds.CONDList(sigfeats)
+
+ ref_latents = kwargs.get("reference_latents", None)
+ if ref_latents is not None:
+ latents = []
+ for lat in ref_latents:
+ latents.append(self.process_latent_in(lat))
+ out['ref_latents'] = comfy.conds.CONDList(latents)
+
+ ref_contexts = kwargs.get("reference_latents_text_embeds", None)
+ if ref_contexts is not None:
+ out['ref_contexts'] = comfy.conds.CONDList(ref_contexts)
+
+ return out
+
+ def extra_conds_shapes(self, **kwargs):
+ out = {}
+ ref_latents = kwargs.get("reference_latents", None)
+ if ref_latents is not None:
+ out['ref_latents'] = list([1, 16, sum(map(lambda a: math.prod(a.size()[2:]), ref_latents))])
return out
class WAN21(BaseModel):
diff --git a/comfy/model_detection.py b/comfy/model_detection.py
index aff5a50b9..42884f797 100644
--- a/comfy/model_detection.py
+++ b/comfy/model_detection.py
@@ -446,6 +446,9 @@ def detect_unet_config(state_dict, key_prefix, metadata=None):
dit_config["time_scale"] = 1000.0
if '{}cap_pad_token'.format(key_prefix) in state_dict_keys:
dit_config["pad_tokens_multiple"] = 32
+ sig_weight = state_dict.get('{}siglip_embedder.0.weight'.format(key_prefix), None)
+ if sig_weight is not None:
+ dit_config["siglip_feat_dim"] = sig_weight.shape[0]
return dit_config
diff --git a/comfy_extras/nodes_zimage.py b/comfy_extras/nodes_zimage.py
new file mode 100644
index 000000000..2ee3c43b1
--- /dev/null
+++ b/comfy_extras/nodes_zimage.py
@@ -0,0 +1,88 @@
+import node_helpers
+from typing_extensions import override
+from comfy_api.latest import ComfyExtension, io
+import math
+import comfy.utils
+
+
+class TextEncodeZImageOmni(io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="TextEncodeZImageOmni",
+ category="advanced/conditioning",
+ is_experimental=True,
+ inputs=[
+ io.Clip.Input("clip"),
+ io.ClipVision.Input("image_encoder", optional=True),
+ io.String.Input("prompt", multiline=True, dynamic_prompts=True),
+ io.Boolean.Input("auto_resize_images", default=True),
+ io.Vae.Input("vae", optional=True),
+ io.Image.Input("image1", optional=True),
+ io.Image.Input("image2", optional=True),
+ io.Image.Input("image3", optional=True),
+ ],
+ outputs=[
+ io.Conditioning.Output(),
+ ],
+ )
+
+ @classmethod
+ def execute(cls, clip, prompt, image_encoder=None, auto_resize_images=True, vae=None, image1=None, image2=None, image3=None) -> io.NodeOutput:
+ ref_latents = []
+ images = list(filter(lambda a: a is not None, [image1, image2, image3]))
+
+ prompt_list = []
+ template = None
+ if len(images) > 0:
+ prompt_list = ["<|im_start|>user\n<|vision_start|>"]
+ prompt_list += ["<|vision_end|><|vision_start|>"] * (len(images) - 1)
+ prompt_list += ["<|vision_end|><|im_end|>"]
+ template = "<|vision_end|>{}<|im_end|>\n<|im_start|>assistant\n<|vision_start|>"
+
+ encoded_images = []
+
+ for i, image in enumerate(images):
+ if image_encoder is not None:
+ encoded_images.append(image_encoder.encode_image(image))
+
+ if vae is not None:
+ if auto_resize_images:
+ samples = image.movedim(-1, 1)
+ total = int(1024 * 1024)
+ scale_by = math.sqrt(total / (samples.shape[3] * samples.shape[2]))
+ width = round(samples.shape[3] * scale_by / 8.0) * 8
+ height = round(samples.shape[2] * scale_by / 8.0) * 8
+
+ image = comfy.utils.common_upscale(samples, width, height, "area", "disabled").movedim(1, -1)
+ ref_latents.append(vae.encode(image))
+
+ tokens = clip.tokenize(prompt, llama_template=template)
+ conditioning = clip.encode_from_tokens_scheduled(tokens)
+
+ extra_text_embeds = []
+ for p in prompt_list:
+ tokens = clip.tokenize(p, llama_template="{}")
+ text_embeds = clip.encode_from_tokens_scheduled(tokens)
+ extra_text_embeds.append(text_embeds[0][0])
+
+ if len(ref_latents) > 0:
+ conditioning = node_helpers.conditioning_set_values(conditioning, {"reference_latents": ref_latents}, append=True)
+ if len(encoded_images) > 0:
+ conditioning = node_helpers.conditioning_set_values(conditioning, {"clip_vision_outputs": encoded_images}, append=True)
+ if len(extra_text_embeds) > 0:
+ conditioning = node_helpers.conditioning_set_values(conditioning, {"reference_latents_text_embeds": extra_text_embeds}, append=True)
+
+ return io.NodeOutput(conditioning)
+
+
+class ZImageExtension(ComfyExtension):
+ @override
+ async def get_node_list(self) -> list[type[io.ComfyNode]]:
+ return [
+ TextEncodeZImageOmni,
+ ]
+
+
+async def comfy_entrypoint() -> ZImageExtension:
+ return ZImageExtension()
diff --git a/nodes.py b/nodes.py
index cba8eacc2..ea5d6e525 100644
--- a/nodes.py
+++ b/nodes.py
@@ -2373,6 +2373,7 @@ async def init_builtin_extra_nodes():
"nodes_kandinsky5.py",
"nodes_wanmove.py",
"nodes_image_compare.py",
+ "nodes_zimage.py",
]
import_failed = []