The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
feat: ming-image support (#16482)
Jukka Seppänen3b4c0b0e457c
diff --git a/comfy/latent_formats.py b/comfy/latent_formats.pyindex c40d8fb10..44141146e 100644--- a/comfy/latent_formats.py+++ b/comfy/latent_formats.py@@ -776,6 +776,32 @@ class Wan21(LatentFormat):latents_std = self.latents_std.to(latent.device, latent.dtype)return latent * latents_std / self.scale_factor + latents_mean+class MingImage(LatentFormat):+ latent_channels = 16+ latent_dimensions = 3+ temporal_downscale_ratio = 4+ scale_factor = 8.0064++ latent_rgb_factors = [+ [ 0.0028, 0.4359, 1.6986],+ [-0.3094, -0.3620, -0.0783],+ [ 0.9806, 0.5697, 0.1953],+ [ 1.0482, 1.0832, 0.4385],+ [ 0.0648, -0.0981, -0.2756],+ [ 0.8954, -0.0321, -0.2461],+ [-0.8438, -1.0825, -0.4071],+ [-0.4610, 0.2319, 0.1578],+ [-1.0171, -0.9257, -1.3410],+ [-0.6377, 0.1949, 0.2250],+ [ 0.0362, 0.6733, -0.0450],+ [-0.0530, 0.3718, 0.7762],+ [-0.9903, 0.2123, 0.4190],+ [-0.2607, -0.7047, -0.1375],+ [ 0.8319, 0.1645, 0.6854],+ [ 0.7366, 0.4105, 0.6899],+ ]+ latent_rgb_factors_bias = [-0.1048, -0.1049, -0.1874]+class Wan22(Wan21):latent_channels = 48latent_dimensions = 3diff --git a/comfy/ldm/lumina/model.py b/comfy/ldm/lumina/model.pyindex cdf03b2b5..62ccdad11 100644--- a/comfy/ldm/lumina/model.py+++ b/comfy/ldm/lumina/model.py@@ -460,6 +460,7 @@ class NextDiT(nn.Module):z_image_modulation=False,time_scale=1.0,pad_tokens_multiple=None,+ masked_pad_multiple=None,clip_text_dim=None,siglip_feat_dim=None,image_model=None,@@ -476,6 +477,7 @@ class NextDiT(nn.Module):self.patch_size = patch_sizeself.time_scale = time_scaleself.pad_tokens_multiple = pad_tokens_multiple+ self.masked_pad_multiple = masked_pad_multipleself.x_embedder = operation_settings.get("operations").Linear(in_features=patch_size * patch_size * in_channels,@@ -627,33 +629,30 @@ class NextDiT(nn.Module):self.n_heads = n_headsdef unpatchify(- self, x: torch.Tensor, img_size: List[Tuple[int, int]], cap_size: List[int], return_tensor=False+ self, x: torch.Tensor, img_size: List[Tuple[int, int]], cap_size: List[int], return_tensor=False, frames=None) -> List[torch.Tensor]:"""x: (N, T, patch_size**2 * C)- imgs: (N, H, W, C)+ imgs: (N, H, W, C), or (N, C, F, H, W) with frames"""pH = pW = self.patch_sizeimgs = []for i in range(x.size(0)):H, W = img_size[i]begin = cap_size[i]- end = begin + (H // pH) * (W // pW)- imgs.append(- x[i][begin:end]- .view(H // pH, W // pW, pH, pW, self.out_channels)- .permute(4, 0, 2, 1, 3)- .flatten(3, 4)- .flatten(1, 2)- )+ end = begin + (frames or 1) * (H // pH) * (W // pW)+ img = x[i][begin:end].view(frames or 1, H // pH, W // pW, pH, pW, self.out_channels).permute(5, 0, 1, 3, 2, 4).flatten(4, 5).flatten(2, 3)+ imgs.append(img if frames else img.squeeze(1))if return_tensor:imgs = torch.stack(imgs, dim=0)return imgs- def embed_cap(self, cap_feats=None, offset=0, bsz=1, device=None, dtype=None):+ def embed_cap(self, cap_feats=None, offset=0, bsz=1, device=None, dtype=None, cap_extra=None):if cap_feats is not None:cap_feats = self.cap_embedder(cap_feats)+ if cap_extra is not None:+ cap_feats = torch.cat((cap_feats, cap_extra), dim=1)cap_feats_len = cap_feats.shape[1]if self.pad_tokens_multiple is not None:cap_feats, _ = pad_zimage(cap_feats, self.cap_pad_token, self.pad_tokens_multiple)@@ -667,14 +666,17 @@ class NextDiT(nn.Module):freqs_cis = (self.rope_embedder(cap_pos_ids).movedim(1, 2),)return embeds, freqs_cis, cap_feats_len- def embed_all(self, x, cap_feats=None, siglip_feats=None, offset=0, omni=False, transformer_options={}):+ def embed_all(self, x, cap_feats=None, siglip_feats=None, offset=0, omni=False, transformer_options={}, cap_extra=None, ref_frames=[]):bsz = 1pH = pW = self.patch_sizedevice = x.device- embeds, freqs_cis, cap_feats_len = self.embed_cap(cap_feats, offset=offset, bsz=bsz, device=device, dtype=x.dtype)+ embeds, freqs_cis, cap_feats_len = self.embed_cap(cap_feats, offset=offset, bsz=bsz, device=device, dtype=x.dtype, cap_extra=cap_extra)if (not omni) or self.siglip_embedder is None:- cap_feats_len = embeds[0].shape[1] + offset+ cap_feats_len = embeds[0].shape[1]+ if self.masked_pad_multiple is not None: # zero-masked context padding only shifts the image positions+ cap_feats_len = -(-cap_feats_len // self.masked_pad_multiple) * self.masked_pad_multiple+ cap_feats_len += offsetembeds += (None,)freqs_cis += (None,)else:@@ -702,9 +704,17 @@ class NextDiT(nn.Module):embeds += (siglip_feats,)freqs_cis += (self.rope_embedder(siglip_pos_ids).movedim(1, 2),)- B, C, H, W = x.shape- x = self.x_embedder(x.view(B, C, H // pH, pH, W // pW, pW).permute(0, 2, 4, 3, 5, 1).flatten(3).flatten(1, 2))- x_pos_ids = pos_ids_x(cap_feats_len + 1, H // pH, W // pW, bsz, device, transformer_options=transformer_options)+ if x.ndim == 4:+ x = x.unsqueeze(2)+ B, C, F, H, W = x.shape+ x = self.x_embedder(x.view(B, C, F, H // pH, pH, W // pW, pW).permute(0, 2, 3, 5, 4, 6, 1).flatten(4).flatten(1, 3))+ x_pos_ids = torch.cat([pos_ids_x(cap_feats_len + 1 + f, H // pH, W // pW, bsz, device, transformer_options=transformer_options) for f in range(F)], dim=1)+ for f, ref in enumerate(ref_frames): # Ming-Image: clean reference frames follow the target frames on the t axis+ ref = comfy.ldm.common_dit.pad_to_patch_size(ref, (pH, pW))+ rH, rW = ref.shape[-2], ref.shape[-1]+ ref = self.x_embedder(ref.view(ref.shape[0], C, rH // pH, pH, rW // pW, pW).permute(0, 2, 4, 3, 5, 1).flatten(3).flatten(1, 2))+ x = torch.cat((x, comfy.utils.repeat_to_batch_size(ref, B)), dim=1)+ x_pos_ids = torch.cat((x_pos_ids, pos_ids_x(cap_feats_len + 1 + F + f, rH // pH, rW // pW, bsz, device, transformer_options=transformer_options)), dim=1)if self.pad_tokens_multiple is not None:x, pad_extra = pad_zimage(x, self.x_pad_token, self.pad_tokens_multiple)x_pos_ids = torch.nn.functional.pad(x_pos_ids, (0, 0, 0, pad_extra))@@ -715,7 +725,7 @@ class NextDiT(nn.Module):def patchify_and_embed(- self, x: torch.Tensor, cap_feats: torch.Tensor, cap_mask: torch.Tensor, t: torch.Tensor, num_tokens, ref_latents=[], ref_contexts=[], siglip_feats=[], transformer_options={}+ self, x: torch.Tensor, cap_feats: torch.Tensor, cap_mask: torch.Tensor, t: torch.Tensor, num_tokens, ref_latents=[], ref_contexts=[], siglip_feats=[], direct_context=None, ref_frames=[], transformer_options={}) -> Tuple[torch.Tensor, torch.Tensor, List[Tuple[int, int]], List[int], torch.Tensor]:bsz = x.shape[0]cap_mask = None # TODO?@@ -749,7 +759,7 @@ class NextDiT(nn.Module):H, W = x.shape[-2], x.shape[-1]img_sizes = [(H, W)] * bsz- out = self.embed_all(x, cap_feats, main_siglip, offset=start_t, omni=omni, transformer_options=transformer_options)+ out = self.embed_all(x, cap_feats, main_siglip, offset=start_t, omni=omni, transformer_options=transformer_options, cap_extra=direct_context, ref_frames=ref_frames)img_len = out[0][-1].shape[1]cap_len = out[0][0].shape[1]for i, e in enumerate(out[0]):@@ -822,7 +832,7 @@ class NextDiT(nn.Module):).execute(x, timesteps, context, num_tokens, attention_mask, **kwargs)# def forward(self, x, t, cap_feats, cap_mask):- def _forward(self, x, timesteps, context, num_tokens, attention_mask=None, ref_latents=[], ref_contexts=[], siglip_feats=[], transformer_options={}, **kwargs):+ def _forward(self, x, timesteps, context, num_tokens, attention_mask=None, ref_latents=[], ref_contexts=[], siglip_feats=[], direct_context=None, ref_frames=[], transformer_options={}, **kwargs):omni = len(ref_latents) > 0if omni:timesteps = torch.cat([timesteps * 0, timesteps], dim=0)@@ -830,8 +840,9 @@ class NextDiT(nn.Module):t = 1.0 - timestepscap_feats = contextcap_mask = attention_mask- bs, c, h, w = x.shape- x = comfy.ldm.common_dit.pad_to_patch_size(x, (self.patch_size, self.patch_size))+ frames = x.shape[2] if x.ndim == 5 else None # Ming-Image latents carry a frame axis: the target, or composite + layers for Ming-Image-Layer+ h, w = x.shape[-2], x.shape[-1]+ x = comfy.ldm.common_dit.pad_to_patch_size(x, (1, self.patch_size, self.patch_size) if frames else (self.patch_size, self.patch_size))"""Forward pass of NextDiT.t: (N,) tensor of diffusion timesteps@@ -852,7 +863,7 @@ class NextDiT(nn.Module):patches = transformer_options.get("patches", {})x_is_tensor = isinstance(x, torch.Tensor)- img, mask, img_size, cap_size, freqs_cis, timestep_zero_index = self.patchify_and_embed(x, cap_feats, cap_mask, adaln_input, num_tokens, ref_latents=ref_latents, ref_contexts=ref_contexts, siglip_feats=siglip_feats, transformer_options=transformer_options)+ img, mask, img_size, cap_size, freqs_cis, timestep_zero_index = self.patchify_and_embed(x, cap_feats, cap_mask, adaln_input, num_tokens, ref_latents=ref_latents, ref_contexts=ref_contexts, siglip_feats=siglip_feats, direct_context=direct_context, ref_frames=ref_frames, transformer_options=transformer_options)freqs_cis = freqs_cis.to(img.device)transformer_options["total_blocks"] = len(self.layers)@@ -870,7 +881,7 @@ class NextDiT(nn.Module):img[:, :cap_size[0]] = out["txt"]img = self.final_layer(img, adaln_input, timestep_zero_index=timestep_zero_index)- img = self.unpatchify(img, img_size, cap_size, return_tensor=x_is_tensor)[:, :, :h, :w]+ img = self.unpatchify(img, img_size, cap_size, return_tensor=x_is_tensor, frames=frames)[..., :h, :w]return -imgdiff --git a/comfy/model_base.py b/comfy/model_base.pyindex f4d3ac838..4f731e7cd 100644--- a/comfy/model_base.py+++ b/comfy/model_base.py@@ -1559,6 +1559,17 @@ class Lumina2(BaseModel):out['ref_latents'] = list([1, 16, sum(map(lambda a: math.prod(a.size()[2:]), ref_latents))])return out+class MingImage(Lumina2):+ def extra_conds(self, **kwargs):+ ref_latents = kwargs.pop("reference_latents", None)+ out = super().extra_conds(**kwargs)+ direct_context = kwargs.get("direct_context", None)+ if direct_context is not None:+ out['direct_context'] = comfy.conds.CONDRegular(direct_context)+ if ref_latents is not None:+ out['ref_frames'] = comfy.conds.CONDList([self.process_latent_in(lat)[:, :, f] for lat in ref_latents for f in range(lat.shape[2])])+ return out+class ZImagePixelSpace(Lumina2):def __init__(self, model_config, model_type=ModelType.FLOW, device=None):BaseModel.__init__(self, model_config, model_type, device=device, unet_model=comfy.ldm.lumina.model.NextDiTPixelSpace)diff --git a/comfy/model_detection.py b/comfy/model_detection.pyindex 1f2342647..6a1d288ec 100644--- a/comfy/model_detection.py+++ b/comfy/model_detection.py@@ -609,13 +609,19 @@ def detect_unet_config(state_dict, key_prefix, metadata=None):dit_config["allow_fp16"] = torch.std(state_dict['{}layers.{}.ffn_norm1.weight'.format(key_prefix, dit_config["n_layers"] - 2)], unbiased=False).item() < 0.42except Exception:pass+ dec_cond_key = '{}dec_net.cond_embed.weight'.format(key_prefix)if '{}cap_pad_token'.format(key_prefix) in state_dict_keys:dit_config["pad_tokens_multiple"] = 32+ # Ming-Image: Z-Image keys. Design ships without the learned pad tokens; Layer has them and only its repack metadata tells it apart.+ ming_metadata = metadata is not None and "config" in metadata and json.loads(metadata["config"]).get("transformer", {}).get("image_model") == "ming_image"+ if ming_metadata or ("pad_tokens_multiple" not in dit_config and dec_cond_key not in state_dict_keys):+ dit_config["image_model"] = "ming_image"+ if "pad_tokens_multiple" not in dit_config:+ dit_config["masked_pad_multiple"] = 32sig_weight = state_dict.get('{}siglip_embedder.0.weight'.format(key_prefix), None)if sig_weight is not None:dit_config["siglip_feat_dim"] = sig_weight.shape[0]- dec_cond_key = '{}dec_net.cond_embed.weight'.format(key_prefix)if dec_cond_key in state_dict_keys: # pixel-space variantdit_config["image_model"] = "zimage_pixel"# patch_size and in_channels are derived from x_embedder:diff --git a/comfy/ops.py b/comfy/ops.pyindex f4ac760c9..860e09ea0 100644--- a/comfy/ops.py+++ b/comfy/ops.py@@ -19,6 +19,7 @@import torchimport loggingimport contextlib+import dataclassesimport inspectimport comfy.model_managementfrom comfy.cli_args import args, PerformanceFeature@@ -1549,8 +1550,18 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_precdef _apply(self, fn, recurse=True):return _quantized_apply(self, fn, recurse)- def _load_from_state_dict(self, *args):- _load_quantized_module(self, super()._load_from_state_dict, *args, load_extra_params=False)+ def _load_from_state_dict(self, state_dict, prefix, *args):+ layer_conf = state_dict.get(f"{prefix}comfy_quant", None)+ quant_format = json.loads(layer_conf.numpy().tobytes()).get("format") if layer_conf is not None else None+ scale = state_dict.get(f"{prefix}weight_scale", None)+ if quant_format == "asym_w4a8_int8" or (quant_format == "int8_tensorwise" and scale is not None and scale.ndim == 3):+ # per-row scaled layouts are 2-D only: keep the bank as [E * out, in] and slice rows per expert+ for name in ("weight", "weight_scale", "weight_s_rel", "weight_s_channel"):+ t = state_dict.get(f"{prefix}{name}", None)+ if t is not None and t.ndim > 1 and t.shape[0] == self.num_experts:+ state_dict[f"{prefix}{name}"] = t.reshape(self.num_experts * t.shape[1], *t.shape[2:])+ self._orig_shape = (self.num_experts * self.out_features, self.in_features)+ _load_quantized_module(self, super()._load_from_state_dict, state_dict, prefix, *args, load_extra_params=False)def expert_weight(self, i: int):"""Expert i's weight (Tensor or per-expert QuantizedTensor view)."""@@ -1558,12 +1569,38 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_precreturn self._expert_qt_from(self.weight, i)return self.weight[i]+ def _cast_bank(self, input):+ # A quantized bank stays quantized: the layouts dequantize one [out, in] matrix at a time, per expert.+ if isinstance(self.weight, QuantizedTensor):+ return CastBiasWeightContext(self, input=None, dtype=self.weight.dtype, device=input.device, bias_dtype=input.dtype, offloadable=True)+ return CastBiasWeightContext(self, input, offloadable=True)++ def _dequantize_bank(self, weight, dtype):+ # in expert chunks: the W4A8 dequantize kernel allocates over twice its output in temporaries+ flat = QuantizedTensor(weight._qdata, weight._layout_cls, dataclasses.replace(weight._params, orig_dtype=dtype))+ out = torch.empty((self.num_experts, self.out_features, self.in_features), dtype=dtype, device=weight.device)+ step = max(1, (256 << 20) // (self.out_features * self.in_features * out.element_size()))+ for first in range(0, self.num_experts, step):+ last = min(first + step, self.num_experts)+ out[first:last] = self._bank_rows(flat, first, last).dequantize().view(last - first, self.out_features, self.in_features)+ return out++ def _bank_rows(self, weight: QuantizedTensor, first: int, last: int) -> QuantizedTensor:+ """Experts [first, last) of a flat [E * out, in] bank as one QuantizedTensor."""+ params = weight._params+ rows = slice(first * self.out_features, last * self.out_features)+ per_row = {f.name: getattr(params, f.name)[rows] for f in dataclasses.fields(params) if torch.is_tensor(getattr(params, f.name)) and getattr(params, f.name).ndim >= 1 and getattr(params, f.name).shape[0] == weight._qdata.shape[0]}+ return QuantizedTensor(weight._qdata[rows], weight._layout_cls, dataclasses.replace(params, orig_shape=((last - first) * self.out_features, self.in_features), **per_row))+@contextlib.contextmanagerdef bank_resident(self, input):"""Cast the whole bank once; expert_linear inside reuses the cast.Not re-entrant — do not nest calls on the same instance."""- with CastBiasWeightContext(self, input, offloadable=True) as self._resident_bank:+ with self._cast_bank(input) as (weight, bias):+ if self._full_precision_mm and isinstance(weight, QuantizedTensor) and weight._qdata.ndim == 2: # flat per-row banks; 3-D banks dequantize per expert+ weight = self._dequantize_bank(weight, input.dtype)+ self._resident_bank = (weight, bias)try:yield selffinally:@@ -1575,14 +1612,14 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_precif resident is not None:weight, bias = residentreturn self._expert_linear_impl(input, weight, bias, i)- with CastBiasWeightContext(self, input, offloadable=True) as (weight, bias):+ with self._cast_bank(input) as (weight, bias):return self._expert_linear_impl(input, weight, bias, i)def _expert_linear_impl(self, input, weight, bias, i):if isinstance(weight, QuantizedTensor):qw = self._expert_qt_from(weight, i)else:- qw = weight[i]+ qw = cast_to_input(weight[i], input, copy=False)b = cast_to_input(bias[i], input, copy=False) if bias is not None else Noneif isinstance(qw, QuantizedTensor):@@ -1594,12 +1631,13 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_precif use_fast:qin = QuantizedTensor.from_float(input, self.layout_type)return torch.nn.functional.linear(qin, qw, b)- out = input @ qw.dequantize().t()- return out + b if b is not None else out+ qw = cast_to_input(qw.dequantize(), input, copy=False)return torch.nn.functional.linear(input, qw, b)def _expert_qt_from(self, weight: QuantizedTensor, i: int) -> QuantizedTensor:"""Build a per-expert QuantizedTensor by indexing into a resident bank."""+ if weight._qdata.ndim == 2:+ return self._bank_rows(weight, i, i + 1)params = weight._paramskwargs = {"scale": params.scale[i] if params.scale.dim() else params.scale,@@ -1610,6 +1648,8 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_preckwargs["block_scale"] = params.block_scale[i]if hasattr(params, "quant_group_size"):kwargs["quant_group_size"] = params.quant_group_size+ if hasattr(params, "convrot"):+ kwargs["convrot"] = params.convrotif hasattr(params, "convrot_groupsize"):kwargs["convrot_groupsize"] = params.convrot_groupsizeif hasattr(params, "linear_dtype"):diff --git a/comfy/sd.py b/comfy/sd.pyindex ea42f6a47..104caae9a 100644--- a/comfy/sd.py+++ b/comfy/sd.py@@ -67,6 +67,7 @@ import comfy.text_encoders.qwen_imageimport comfy.text_encoders.qwen_image21import comfy.text_encoders.hunyuan_imageimport comfy.text_encoders.z_image+import comfy.text_encoders.ming_imageimport comfy.text_encoders.krea2import comfy.text_encoders.mage_flowimport comfy.text_encoders.ideogram4@@ -1640,9 +1641,12 @@ class TEModel(Enum):QWEN3VL_8B = 35GEMMA_4_12B = 36QWEN3VL_32B = 37+ MING_IMAGE = 38def detect_te_model(sd):+ if "thinker.layers.1.mlp.image_gate.proj.weight" in sd:+ return TEModel.MING_IMAGEif "text_model.encoder.layers.30.mlp.fc1.weight" in sd:return TEModel.CLIP_Gif "text_model.encoder.layers.22.mlp.fc1.weight" in sd:@@ -1902,6 +1906,11 @@ def load_text_encoder_state_dicts(state_dicts=[], embedding_directory=None, clipclip_target.clip = comfy.text_encoders.flux.flux2_te(**llama_detect(clip_data), pruned=te_model == TEModel.MISTRAL3_24B_PRUNED_FLUX2)clip_target.tokenizer = comfy.text_encoders.flux.Flux2Tokenizertokenizer_data["tekken_model"] = clip_data[0].get("tekken_model", None)+ elif te_model == TEModel.MING_IMAGE:+ tokenizer_data["tokenizer_json"] = clip_data[0].get("tokenizer_json", None)+ quant = comfy.utils.detect_layer_quantization(clip_data[0], "")+ clip_target.clip = comfy.text_encoders.ming_image.te(dtype_llama=clip_data[0]["thinker.norm.weight"].dtype, llama_quantization_metadata=quant)+ clip_target.tokenizer = comfy.text_encoders.ming_image.MingImageTokenizerelif te_model == TEModel.GPT_OSS_20B:clip_target.clip = comfy.text_encoders.gpt_oss.lens_te(**llama_detect(clip_data))clip_target.tokenizer = comfy.text_encoders.gpt_oss.LensTokenizer@@ -2346,7 +2355,7 @@ def load_diffusion_model_state_dict(sd, model_options={}, metadata=None, disableelse:new_sd = model_detection.convert_diffusers_mmdit(sd, "")if new_sd is not None: #diffusers mmdit- model_config = model_detection.model_config_from_unet(new_sd, "")+ model_config = model_detection.model_config_from_unet(new_sd, "", metadata=metadata)if model_config is None:return Noneelse: #diffusers unetdiff --git a/comfy/supported_models.py b/comfy/supported_models.pyindex b5cc18736..b4ca22715 100644--- a/comfy/supported_models.py+++ b/comfy/supported_models.py@@ -27,6 +27,7 @@ import comfy.text_encoders.qwen_image21import comfy.text_encoders.hunyuan_imageimport comfy.text_encoders.kandinsky5import comfy.text_encoders.z_image+import comfy.text_encoders.ming_imageimport comfy.text_encoders.ideogram4import comfy.text_encoders.booguimport comfy.text_encoders.krea2@@ -1242,6 +1243,24 @@ class ZImagePixelSpace(ZImage):def get_model(self, state_dict, prefix="", device=None):return model_base.ZImagePixelSpace(self, device=device)+class MingImage(ZImage):+ unet_config = {+ "image_model": "ming_image",+ }++ sampling_settings = {+ "multiplier": 1.0,+ "shift": 3.16, # reference dynamic shift at the 1024 bucket+ }++ latent_format = latent_formats.MingImage++ def get_model(self, state_dict, prefix="", device=None):+ return model_base.MingImage(self, device=device)++ def clip_target(self, state_dict={}):+ return supported_models_base.ClipTarget(comfy.text_encoders.ming_image.MingImageTokenizer, comfy.text_encoders.ming_image.te())+class PixelDiTT2I(supported_models_base.BASE):unet_config = {"image_model": "pixeldit_t2i",@@ -2602,6 +2621,7 @@ models = [CosmosT2IPredict2,CosmosI2VPredict2,ZImagePixelSpace,+ MingImage,ZImage,PiD,PixelDiTT2I,diff --git a/comfy/text_encoders/gpt_oss.py b/comfy/text_encoders/gpt_oss.pyindex 066796b6a..744ea6c92 100644--- a/comfy/text_encoders/gpt_oss.py+++ b/comfy/text_encoders/gpt_oss.py@@ -13,7 +13,7 @@ import torch.nn.functional as Fimport comfy.opsfrom comfy import sd1_clipfrom comfy.ldm.modules.attention import optimized_attention_for_device-from comfy.text_encoders.llama import RMSNorm, apply_rope+from comfy.text_encoders.llama import RMSNorm, apply_rope, moe_experts_forward@dataclass@@ -205,32 +205,7 @@ class GptOssExperts(nn.Module):return torch.addcmul(glu, up, glu)def forward(self, hidden_states: torch.Tensor, router_indices: torch.Tensor, routing_weights: torch.Tensor) -> torch.Tensor:- N = hidden_states.shape[0]- top_k = router_indices.shape[-1]- H = hidden_states.shape[-1]-- per_pair = torch.zeros((N * top_k, H), dtype=hidden_states.dtype, device=hidden_states.device)-- expert_mask = F.one_hot(router_indices, num_classes=self.num_experts).permute(2, 1, 0)- expert_hit = torch.greater(expert_mask.sum(dim=(-1, -2)), 0).nonzero()-- with self.gate_up_proj.bank_resident(hidden_states) as gate_up_bank, \- self.down_proj.bank_resident(hidden_states) as down_bank:- for ei in expert_hit:- expert_idx = int(ei.item())- top_k_pos, token_idx = torch.where(expert_mask[expert_idx])- current = hidden_states[token_idx]-- gate_up = gate_up_bank.expert_linear(current, expert_idx)- gated = self._apply_gate(gate_up)- expert_out = down_bank.expert_linear(gated, expert_idx)-- weighted = expert_out * routing_weights[token_idx, top_k_pos, None]-- flat_idx = token_idx * top_k + top_k_pos- per_pair[flat_idx] = weighted.to(per_pair.dtype)-- return per_pair.view(N, top_k, H).sum(dim=1)+ return moe_experts_forward(hidden_states, router_indices, routing_weights, self.num_experts, self.gate_up_proj, self.down_proj, self._apply_gate)class GptOssMLP(nn.Module):diff --git a/comfy/text_encoders/llama.py b/comfy/text_encoders/llama.pyindex 96577011a..ba55fbf14 100644--- a/comfy/text_encoders/llama.py+++ b/comfy/text_encoders/llama.py@@ -558,6 +558,29 @@ def precompute_freqs_cis(head_dim, position_ids, theta, rope_scale=None, rope_direturn out+def moe_experts_forward(x, topk_idx, topk_weight, num_experts, gate_up_proj, down_proj, activation):+ num_tokens, top_k = topk_idx.shape+ # group the (token, slot) assignments by expert: one host sync per call instead of two per expert+ order = torch.argsort(topk_idx.reshape(-1))+ counts = torch.bincount(topk_idx.reshape(-1), minlength=num_experts).tolist()+ sorted_x = x[order // top_k]+ weight = topk_weight.reshape(-1)[order].unsqueeze(1)+ sorted_out = torch.empty_like(sorted_x)++ start = 0+ with gate_up_proj.bank_resident(x) as gate_up_bank, down_proj.bank_resident(x) as down_bank:+ for expert_idx, n in enumerate(counts):+ if n == 0:+ continue+ gated = activation(gate_up_bank.expert_linear(sorted_x[start:start + n], expert_idx))+ sorted_out[start:start + n] = (down_bank.expert_linear(gated, expert_idx) * weight[start:start + n]).to(sorted_out.dtype)+ start += n++ out = torch.empty_like(sorted_out)+ out[order] = sorted_out+ return out.view(num_tokens, top_k, -1).sum(dim=1)++def rope_matrix(freqs_cis):if torch.is_tensor(freqs_cis):return freqs_cisdiff --git a/comfy/text_encoders/ming_image.py b/comfy/text_encoders/ming_image.pynew file mode 100644index 000000000..cf85fee6f--- /dev/null+++ b/comfy/text_encoders/ming_image.py@@ -0,0 +1,349 @@+from dataclasses import dataclass++import torch+import torch.nn as nn+import torch.nn.functional as F+from tokenizers import Tokenizer++import comfy.ops+from comfy import sd1_clip+from comfy.ldm.modules.attention import optimized_attention_for_device+from comfy.text_encoders.llama import MLP, RMSNorm, TransformerBlock, moe_experts_forward, precompute_freqs_cis+from comfy.text_encoders.qwen35 import apply_partial_rope+from comfy.text_encoders import qwen_vl++IMAGE_PATCH_TOKEN = 157157+IMAGE_BLOCK = "<image><imagePatch></image>"+++@dataclass+class BailingMoeV2Config:+ vocab_size: int = 157184+ hidden_size: int = 2048+ intermediate_size: int = 5120+ moe_intermediate_size: int = 512+ num_hidden_layers: int = 20+ num_attention_heads: int = 16+ num_key_value_heads: int = 4+ head_dim: int = 128+ rope_dim: int = 64+ rope_theta: float = 600000.0+ rms_norm_eps: float = 1e-6+ num_experts: int = 256+ num_experts_per_tok: int = 8+ n_group: int = 8+ topk_group: int = 4+ routed_scaling_factor: float = 2.5+ mlp_activation: str = "silu"+++@dataclass+class MingConnectorConfig:+ hidden_size: int = 1536+ intermediate_size: int = 8960+ num_hidden_layers: int = 28+ num_attention_heads: int = 12+ num_key_value_heads: int = 2+ head_dim: int = 128+ rms_norm_eps: float = 1e-6+ rope_theta: float = 1000000.0+ qkv_bias: bool = True+ rms_norm_add: bool = False+ mlp_activation: str = "silu"+ q_norm = None+ k_norm = None+++class BailingAttention(nn.Module):+ def __init__(self, config, device=None, dtype=None, ops=None):+ super().__init__()+ self.num_heads = config.num_attention_heads+ self.num_kv_heads = config.num_key_value_heads+ self.head_dim = config.head_dim+ self.rope_dim = config.rope_dim+ self.query_key_value = ops.Linear(config.hidden_size, (self.num_heads + 2 * self.num_kv_heads) * self.head_dim, bias=False, device=device, dtype=dtype)+ self.q_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps, device=device, dtype=dtype)+ self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps, device=device, dtype=dtype)+ self.dense = ops.Linear(self.num_heads * self.head_dim, config.hidden_size, bias=False, device=device, dtype=dtype)++ def forward(self, x, attention_mask, freqs_cis, optimized_attention):+ batch, seq_len, _ = x.shape+ qkv = self.query_key_value(x).view(batch, seq_len, -1, self.head_dim).transpose(1, 2)+ xq, xk, xv = qkv.split((self.num_heads, self.num_kv_heads, self.num_kv_heads), dim=1)+ xq = self.q_norm(xq)+ xk = self.k_norm(xk)+ xq, xk = apply_partial_rope(xq, xk, freqs_cis, self.rope_dim)+ out = optimized_attention(xq, xk, xv, self.num_heads, mask=attention_mask, skip_reshape=True, enable_gqa=True)+ return self.dense(out)+++class BailingGate(nn.Module):+ def __init__(self, config, device=None, dtype=None, ops=None):+ super().__init__()+ self.top_k = config.num_experts_per_tok+ self.n_group = config.n_group+ self.topk_group = config.topk_group+ self.routed_scaling_factor = config.routed_scaling_factor+ self.proj = ops.Linear(config.hidden_size, config.num_experts, bias=False, device=device, dtype=dtype)+ self.expert_bias = nn.Parameter(torch.empty(config.num_experts, device=device, dtype=torch.float32))++ def forward(self, x):+ scores = torch.sigmoid(self.proj(x.float()))+ routing = scores + comfy.ops.cast_to_input(self.expert_bias, scores, copy=False)+ num_tokens = routing.shape[0]+ grouped = routing.view(num_tokens, self.n_group, -1)+ group_idx = torch.topk(grouped.topk(2, dim=-1)[0].sum(dim=-1), k=self.topk_group, dim=-1, sorted=False)[1]+ group_mask = torch.zeros(num_tokens, self.n_group, dtype=torch.bool, device=x.device).scatter_(1, group_idx, True)+ topk_idx = torch.topk(grouped.masked_fill(~group_mask.unsqueeze(-1), float("-inf")).view(num_tokens, -1), k=self.top_k, dim=-1, sorted=False)[1]+ topk_weight = torch.gather(scores, dim=1, index=topk_idx)+ return topk_idx, topk_weight / (topk_weight.sum(dim=-1, keepdim=True) + 1e-20) * self.routed_scaling_factor+++class BailingExperts(nn.Module):+ def __init__(self, config, device=None, dtype=None, ops=None):+ super().__init__()+ self.num_experts = config.num_experts+ self.gate_up_proj = ops.MoEExperts(num_experts=config.num_experts, in_features=config.hidden_size, out_features=2 * config.moe_intermediate_size, bias=False, device=device, dtype=dtype)+ self.down_proj = ops.MoEExperts(num_experts=config.num_experts, in_features=config.moe_intermediate_size, out_features=config.hidden_size, bias=False, device=device, dtype=dtype)++ def forward(self, x, topk_idx, topk_weight):+ return moe_experts_forward(x, topk_idx, topk_weight, self.num_experts, self.gate_up_proj, self.down_proj, comfy.ops._swiglu_eager)+++class BailingSparseMoe(nn.Module):+ def __init__(self, config, device=None, dtype=None, ops=None):+ super().__init__()+ self.gate = BailingGate(config, device=device, dtype=dtype, ops=ops)+ self.image_gate = BailingGate(config, device=device, dtype=dtype, ops=ops)+ self.experts = BailingExperts(config, device=device, dtype=dtype, ops=ops)+ self.shared_experts = MLP(config, device=device, dtype=dtype, ops=ops, intermediate_size=config.moe_intermediate_size)++ def forward(self, x, image_mask):+ batch, seq_len, hidden = x.shape+ flat = x.reshape(-1, hidden)+ text_idx, text_weight = self.gate(flat)+ image_idx, image_weight = self.image_gate(flat)+ mask = image_mask.reshape(-1, 1)+ out = self.experts(flat, torch.where(mask, image_idx, text_idx), torch.where(mask, image_weight, text_weight))+ return out.view(batch, seq_len, hidden) + self.shared_experts(x)+++class BailingDecoderLayer(nn.Module):+ def __init__(self, config, index, device=None, dtype=None, ops=None):+ super().__init__()+ self.attention = BailingAttention(config, device=device, dtype=dtype, ops=ops)+ self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps, device=device, dtype=dtype)+ self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps, device=device, dtype=dtype)+ self.dense = index == 0+ if self.dense:+ self.mlp = MLP(config, device=device, dtype=dtype, ops=ops)+ else:+ self.mlp = BailingSparseMoe(config, device=device, dtype=dtype, ops=ops)++ def forward(self, x, attention_mask, freqs_cis, optimized_attention, image_mask):+ x = x + self.attention(self.input_layernorm(x), attention_mask, freqs_cis, optimized_attention)+ h = self.post_attention_layernorm(x)+ if self.dense:+ return x + self.mlp(h)+ return x + self.mlp(h, image_mask)+++class BailingMoeV2(nn.Module):+ def __init__(self, config, device=None, dtype=None, ops=None):+ super().__init__()+ self.config = config+ self.embed_tokens = ops.Embedding(config.vocab_size, config.hidden_size, device=device, dtype=dtype)+ self.layers = nn.ModuleList([BailingDecoderLayer(config, i, device=device, dtype=dtype, ops=ops) for i in range(config.num_hidden_layers)])+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps, device=device, dtype=dtype)++ def freqs_cis(self, seq_len, blocks, device):+ # video_rope: an image block takes one text position, its tokens form an (h, w) grid centered there+ t = torch.arange(seq_len, device=device, dtype=torch.float32)+ for start, gh, gw in blocks:+ t[start + gh * gw:] -= gh * gw - 1+ h = t.clone()+ w = t.clone()+ shift = 0+ for start, gh, gw in blocks:+ end = start + gh * gw+ t[start:end] = start - shift+ h[start:end] = start - shift + (torch.arange(gh, device=device) - (gh - 1) // 2).repeat_interleave(gw)+ w[start:end] = start - shift + (torch.arange(gw, device=device) - (gw - 1) // 2).repeat(gh)+ shift += gh * gw - 1+ inv_freq = 1.0 / (self.config.rope_theta ** (torch.arange(0, self.config.rope_dim, 2, device=device, dtype=torch.float32) / self.config.rope_dim))+ freqs = t[:, None] * inv_freq+ freqs[:, 0:24:2] = h[:, None] * inv_freq[0:24:2]+ freqs[:, 1:24:2] = w[:, None] * inv_freq[1:24:2]+ freqs = freqs.unsqueeze(0)+ return torch.cat((freqs, freqs), dim=-1).cos(), freqs.sin(), -freqs.sin()++ def forward(self, x, attention_mask, blocks, capture_layers):+ seq_len = x.shape[1]+ freqs_cis = self.freqs_cis(seq_len, blocks, x.device)+ neg = torch.finfo(x.dtype).min / 4+ mask = torch.full((seq_len, seq_len), neg, dtype=x.dtype, device=x.device).triu_(1)+ mask = mask + (attention_mask == 0).to(x.dtype)[:, None, None, :] * neg+ optimized_attention = optimized_attention_for_device(x.device, mask=True, small_input=True)+ image_mask = torch.zeros(x.shape[:2], dtype=torch.bool, device=x.device)+ for start, gh, gw in blocks:+ image_mask[:, start:start + gh * gw] = True++ captured = []+ for i, layer in enumerate(self.layers):+ if i in capture_layers:+ captured.append(x)+ x = layer(x, mask, freqs_cis, optimized_attention, image_mask)+ x = self.norm(x)+ captured.append(x)+ return x, captured+++class MingConnector(nn.Module):+ def __init__(self, config, device=None, dtype=None, ops=None):+ super().__init__()+ self.config = config+ self.layers = nn.ModuleList([TransformerBlock(config, index=i, device=device, dtype=dtype, ops=ops) for i in range(config.num_hidden_layers)])+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps, device=device, dtype=dtype)++ def forward(self, x):+ position_ids = torch.arange(x.shape[1], device=x.device).unsqueeze(0)+ freqs_cis = precompute_freqs_cis(self.config.head_dim, position_ids, self.config.rope_theta, device=x.device)+ optimized_attention = optimized_attention_for_device(x.device, mask=False, small_input=True)+ for layer in self.layers:+ x, _ = layer(x, attention_mask=None, freqs_cis=freqs_cis, optimized_attention=optimized_attention)+ return self.norm(x)+++class MingImageEncoder(nn.Module):+ capture_layers = (5, 12)++ def __init__(self, config_dict, dtype, device, operations):+ super().__init__()+ config = BailingMoeV2Config(**config_dict)+ connector_config = MingConnectorConfig()+ self.num_layers = config.num_hidden_layers+ self.thinker = BailingMoeV2(config, device=device, dtype=dtype, ops=operations)+ self.vision = qwen_vl.Qwen2VLVisionTransformer(hidden_size=1280, output_hidden_size=8192, intermediate_size=3456, num_heads=16, num_layers=32, device=device, dtype=dtype, ops=operations)+ self.linear_proj = nn.Sequential(+ operations.Linear(8192, config.hidden_size, device=device, dtype=dtype),+ nn.GELU(),+ operations.Linear(config.hidden_size, config.hidden_size, device=device, dtype=dtype),+ )+ self.connector = MingConnector(connector_config, device=device, dtype=dtype, ops=operations)+ self.query_tokens = nn.Parameter(torch.empty(256, config.hidden_size, device=device, dtype=dtype))+ self.proj_in = operations.Linear(config.hidden_size, connector_config.hidden_size, device=device, dtype=dtype)+ self.proj_out = operations.Linear(connector_config.hidden_size, 2560, device=device, dtype=dtype)+ direct_dim = config.hidden_size * (len(self.capture_layers) + 1)+ self.proj_directvlm = nn.Sequential(+ operations.RMSNorm(direct_dim, eps=1e-5, elementwise_affine=True, device=device, dtype=dtype),+ operations.Linear(direct_dim, 3840, device=device, dtype=dtype),+ )++ def get_input_embeddings(self):+ return self.thinker.embed_tokens++ def preprocess_embed(self, embed, device):+ if embed["type"] == "query":+ return self.query_tokens, None+ if embed["type"] == "image":+ pixels, grid = qwen_vl.process_qwen2vl_images(embed["data"], min_pixels=451584, max_pixels=451584)+ feats = self.vision(pixels.to(device, dtype=torch.float32), grid)+ return F.normalize(self.linear_proj(feats), dim=-1), grid+ return None, None++ def forward(self, embeds, attention_mask, embeds_info):+ blocks = [(e["index"], int(e["extra"][0][1]) // 2, int(e["extra"][0][2]) // 2) if e["type"] == "image" else (e["index"], 1, e["size"]) for e in embeds_info]+ start, _, size = blocks[-1]+ hidden, captured = self.thinker(embeds, attention_mask, blocks, self.capture_layers)+ cap_feats = self.proj_out(self.connector(self.proj_in(hidden[:, start:start + size])))+ direct = torch.cat([c[:, :start - 1] for c in captured], dim=-1)+ return cap_feats, self.proj_directvlm(direct)+++class _MingRawTokenizer:+ def __init__(self, tokenizer_json_bytes=None, **kwargs):+ if isinstance(tokenizer_json_bytes, torch.Tensor):+ tokenizer_json_bytes = tokenizer_json_bytes.numpy().tobytes()+ if not isinstance(tokenizer_json_bytes, bytes):+ raise ValueError("The Ming-Image text encoder file must contain the tokenizer_json tensor.")+ self.tokenizer = Tokenizer.from_str(tokenizer_json_bytes.decode("utf-8"))++ @classmethod+ def from_pretrained(cls, tokenizer_data, **kwargs):+ return cls(tokenizer_json_bytes=tokenizer_data, **kwargs)++ def __call__(self, text):+ return {"input_ids": self.tokenizer.encode(text, add_special_tokens=False).ids}++ def get_vocab(self):+ return self.tokenizer.get_vocab()++ def decode(self, ids, **kwargs):+ return self.tokenizer.decode(ids, skip_special_tokens=kwargs.get("skip_special_tokens", False))+++class MingTokenizer(sd1_clip.SDTokenizer):+ def __init__(self, embedding_directory=None, tokenizer_data={}):+ self.tokenizer_json_data = tokenizer_data.get("tokenizer_json", None)+ super().__init__(self.tokenizer_json_data, pad_with_end=False, embedding_directory=embedding_directory, embedding_size=2048, embedding_key='ming_image', tokenizer_class=_MingRawTokenizer, has_start_token=False, has_end_token=False, pad_to_max_length=False, max_length=99999999, min_length=1, pad_token=156892, disable_weights=True, tokenizer_data=tokenizer_data)++ def state_dict(self):+ return {"tokenizer_json": self.tokenizer_json_data}+++class MingImageTokenizer(sd1_clip.SD1Tokenizer):+ def __init__(self, embedding_directory=None, tokenizer_data={}):+ super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, name="ming_image", tokenizer=MingTokenizer)+ self.llama_template = "<role>SYSTEM</role>你是一个友好的AI助手。\n\ndetailed thinking off<|role_end|><role>HUMAN</role>{}<|role_end|><role>ASSISTANT</role>" + IMAGE_BLOCK++ def tokenize_with_weights(self, text, return_word_ids=False, llama_template=None, images=[], **kwargs):+ if llama_template is None:+ llama_template = self.llama_template+ if len(images) > 0: # reference images lead the user turn, separated by blank lines like the vendor processor+ llama_template = llama_template.replace("{}", "\n\n".join([IMAGE_BLOCK] * len(images)) + "\n{}")+ tokens = super().tokenize_with_weights(llama_template.format(text), return_word_ids=return_word_ids, **kwargs)+ images = iter(images)+ for r in tokens["ming_image"]:+ for i in range(len(r)):+ if r[i][0] == IMAGE_PATCH_TOKEN:+ image = next(images, None)+ r[i] = ({"type": "query"} if image is None else {"type": "image", "data": image},) + r[i][1:]+ return tokens+++class MingImageClipModel(sd1_clip.SDClipModel):+ def __init__(self, device="cpu", dtype=None, model_options={}):+ super().__init__(device=device, layer="last", layer_idx=None, textmodel_json_config={}, dtype=dtype, special_tokens={"pad": 156892}, layer_norm_hidden_state=False, model_class=MingImageEncoder, enable_attention_masks=True, return_attention_masks=False, model_options=model_options)++ def forward(self, tokens):+ if self.execution_device is None:+ device = self.transformer.get_input_embeddings().weight.device+ else:+ device = self.execution_device+ embeds, attention_mask, num_tokens, embeds_info = self.process_tokens(tokens, device)+ cap_feats, direct_context = self.transformer(embeds, attention_mask, embeds_info)+ return cap_feats.float(), None, {"direct_context": direct_context.float()}+++class MingImageTEModel(sd1_clip.SD1ClipModel):+ def __init__(self, device="cpu", dtype=None, model_options={}):+ super().__init__(device=device, dtype=dtype, name="ming_image", clip_model=MingImageClipModel, model_options=model_options)++ def memory_estimation_function(self, tokens, device=None):+ # both expert banks of a MoE layer sit in fp32 while it runs, on top of the weights; images expand to a few hundred tokens each+ config = BailingMoeV2Config()+ banks = 3 * config.hidden_size * config.moe_intermediate_size * config.num_experts * 4+ num_tokens = sum(600 if isinstance(t[0], dict) else 1 for batch in tokens.get("ming_image", []) for t in batch)+ return banks + num_tokens * config.hidden_size * 64+++def te(dtype_llama=None, llama_quantization_metadata=None):+ class MingImageTEModel_(MingImageTEModel):+ def __init__(self, device="cpu", dtype=None, model_options={}):+ if dtype_llama is not None:+ dtype = dtype_llama+ model_options = model_options.copy()+ if "custom_operations" not in model_options:+ model_options["custom_operations"] = comfy.ops.mixed_precision_ops(llama_quantization_metadata or {}, dtype, full_precision_mm=True)+ super().__init__(device=device, dtype=dtype, model_options=model_options)+ return MingImageTEModel_diff --git a/comfy_extras/nodes_ming.py b/comfy_extras/nodes_ming.pynew file mode 100644index 000000000..6dce384ed--- /dev/null+++ b/comfy_extras/nodes_ming.py@@ -0,0 +1,54 @@+import comfy.utils+import node_helpers+from typing_extensions import override+from comfy_api.latest import ComfyExtension, io+++class TextEncodeMingImageEdit(io.ComfyNode):+ @classmethod+ def define_schema(cls):+ return io.Schema(+ node_id="TextEncodeMingImageEdit",+ display_name="Text Encode Ming Image Edit",+ category="model/conditioning/ming image",+ inputs=[+ io.Clip.Input("clip"),+ io.Vae.Input("vae", optional=True, tooltip="Without a VAE the images only condition the text encoder through the vision tower."),+ io.String.Input("prompt", multiline=True, dynamic_prompts=True),+ io.Autogrow.Input(+ "images",+ template=io.Autogrow.TemplateNames(+ io.Image.Input("image"),+ names=[f"image_{i}" for i in range(1, 9)],+ min=0,+ ),+ tooltip="Optional reference images, seen by the text encoder and appended to the latent sequence as clean frames. Later images are resized to the first one, and the sampled latent should match its size.",+ ),+ ],+ outputs=[+ io.Conditioning.Output(),+ ],+ )++ @classmethod+ def execute(cls, clip, prompt, images: io.Autogrow.Type, vae=None) -> io.NodeOutput:+ images = [images[name] for name in sorted(images, key=lambda n: int(n.rsplit("_", 1)[-1])) if images[name] is not None]+ tokens = clip.tokenize(prompt, images=[image[:, :, :, :3] for image in images])+ conditioning = clip.encode_from_tokens_scheduled(tokens)+ if vae is not None and len(images) > 0:+ height, width = images[0].shape[1:3] # every reference frame shares the canvas of the first one, as the vendor processor does+ ref_latents = [vae.encode(comfy.utils.common_upscale(image.movedim(-1, 1), width, height, "bilinear", "disabled").movedim(1, -1)) for image in images]+ conditioning = node_helpers.conditioning_set_values(conditioning, {"reference_latents": ref_latents}, append=True)+ return io.NodeOutput(conditioning)+++class MingExtension(ComfyExtension):+ @override+ async def get_node_list(self) -> list[type[io.ComfyNode]]:+ return [+ TextEncodeMingImageEdit,+ ]+++async def comfy_entrypoint() -> MingExtension:+ return MingExtension()diff --git a/nodes.py b/nodes.pyindex 9bbba5a29..1d3fd6b16 100644--- a/nodes.py+++ b/nodes.py@@ -293,6 +293,9 @@ class ConditioningZeroOut:conditioning_scale = d.get("conditioning_scale", None)if conditioning_scale is not None:d["conditioning_scale"] = torch.zeros_like(conditioning_scale)+ direct_context = d.get("direct_context", None)+ if direct_context is not None:+ d["direct_context"] = torch.zeros_like(direct_context)n = [torch.zeros_like(t[0]), d]c.append(n)return (c, )@@ -2487,6 +2490,7 @@ async def init_builtin_extra_nodes():"nodes_seedvr.py","nodes_context_windows.py","nodes_qwen.py",+ "nodes_ming.py","nodes_mage.py","nodes_joyimage.py","nodes_boogu.py",