Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

feat: ming-image support (#16482)

Jukka Seppänen3b4c0b0e457c
diff --git a/comfy/latent_formats.py b/comfy/latent_formats.py
index c40d8fb10..44141146e 100644
--- a/comfy/latent_formats.py
+++ b/comfy/latent_formats.py
@@ -776,6 +776,32 @@ class Wan21(LatentFormat):
latents_std = self.latents_std.to(latent.device, latent.dtype)
return latent * latents_std / self.scale_factor + latents_mean
+class MingImage(LatentFormat):
+ latent_channels = 16
+ latent_dimensions = 3
+ temporal_downscale_ratio = 4
+ scale_factor = 8.0064
+
+ latent_rgb_factors = [
+ [ 0.0028, 0.4359, 1.6986],
+ [-0.3094, -0.3620, -0.0783],
+ [ 0.9806, 0.5697, 0.1953],
+ [ 1.0482, 1.0832, 0.4385],
+ [ 0.0648, -0.0981, -0.2756],
+ [ 0.8954, -0.0321, -0.2461],
+ [-0.8438, -1.0825, -0.4071],
+ [-0.4610, 0.2319, 0.1578],
+ [-1.0171, -0.9257, -1.3410],
+ [-0.6377, 0.1949, 0.2250],
+ [ 0.0362, 0.6733, -0.0450],
+ [-0.0530, 0.3718, 0.7762],
+ [-0.9903, 0.2123, 0.4190],
+ [-0.2607, -0.7047, -0.1375],
+ [ 0.8319, 0.1645, 0.6854],
+ [ 0.7366, 0.4105, 0.6899],
+ ]
+ latent_rgb_factors_bias = [-0.1048, -0.1049, -0.1874]
+
class Wan22(Wan21):
latent_channels = 48
latent_dimensions = 3
diff --git a/comfy/ldm/lumina/model.py b/comfy/ldm/lumina/model.py
index cdf03b2b5..62ccdad11 100644
--- a/comfy/ldm/lumina/model.py
+++ b/comfy/ldm/lumina/model.py
@@ -460,6 +460,7 @@ class NextDiT(nn.Module):
z_image_modulation=False,
time_scale=1.0,
pad_tokens_multiple=None,
+ masked_pad_multiple=None,
clip_text_dim=None,
siglip_feat_dim=None,
image_model=None,
@@ -476,6 +477,7 @@ class NextDiT(nn.Module):
self.patch_size = patch_size
self.time_scale = time_scale
self.pad_tokens_multiple = pad_tokens_multiple
+ self.masked_pad_multiple = masked_pad_multiple
self.x_embedder = operation_settings.get("operations").Linear(
in_features=patch_size * patch_size * in_channels,
@@ -627,33 +629,30 @@ class NextDiT(nn.Module):
self.n_heads = n_heads
def unpatchify(
- self, x: torch.Tensor, img_size: List[Tuple[int, int]], cap_size: List[int], return_tensor=False
+ self, x: torch.Tensor, img_size: List[Tuple[int, int]], cap_size: List[int], return_tensor=False, frames=None
) -> List[torch.Tensor]:
"""
x: (N, T, patch_size**2 * C)
- imgs: (N, H, W, C)
+ imgs: (N, H, W, C), or (N, C, F, H, W) with frames
"""
pH = pW = self.patch_size
imgs = []
for i in range(x.size(0)):
H, W = img_size[i]
begin = cap_size[i]
- end = begin + (H // pH) * (W // pW)
- imgs.append(
- x[i][begin:end]
- .view(H // pH, W // pW, pH, pW, self.out_channels)
- .permute(4, 0, 2, 1, 3)
- .flatten(3, 4)
- .flatten(1, 2)
- )
+ end = begin + (frames or 1) * (H // pH) * (W // pW)
+ img = x[i][begin:end].view(frames or 1, H // pH, W // pW, pH, pW, self.out_channels).permute(5, 0, 1, 3, 2, 4).flatten(4, 5).flatten(2, 3)
+ imgs.append(img if frames else img.squeeze(1))
if return_tensor:
imgs = torch.stack(imgs, dim=0)
return imgs
- def embed_cap(self, cap_feats=None, offset=0, bsz=1, device=None, dtype=None):
+ def embed_cap(self, cap_feats=None, offset=0, bsz=1, device=None, dtype=None, cap_extra=None):
if cap_feats is not None:
cap_feats = self.cap_embedder(cap_feats)
+ if cap_extra is not None:
+ cap_feats = torch.cat((cap_feats, cap_extra), dim=1)
cap_feats_len = cap_feats.shape[1]
if self.pad_tokens_multiple is not None:
cap_feats, _ = pad_zimage(cap_feats, self.cap_pad_token, self.pad_tokens_multiple)
@@ -667,14 +666,17 @@ class NextDiT(nn.Module):
freqs_cis = (self.rope_embedder(cap_pos_ids).movedim(1, 2),)
return embeds, freqs_cis, cap_feats_len
- def embed_all(self, x, cap_feats=None, siglip_feats=None, offset=0, omni=False, transformer_options={}):
+ def embed_all(self, x, cap_feats=None, siglip_feats=None, offset=0, omni=False, transformer_options={}, cap_extra=None, ref_frames=[]):
bsz = 1
pH = pW = self.patch_size
device = x.device
- embeds, freqs_cis, cap_feats_len = self.embed_cap(cap_feats, offset=offset, bsz=bsz, device=device, dtype=x.dtype)
+ embeds, freqs_cis, cap_feats_len = self.embed_cap(cap_feats, offset=offset, bsz=bsz, device=device, dtype=x.dtype, cap_extra=cap_extra)
if (not omni) or self.siglip_embedder is None:
- cap_feats_len = embeds[0].shape[1] + offset
+ cap_feats_len = embeds[0].shape[1]
+ if self.masked_pad_multiple is not None: # zero-masked context padding only shifts the image positions
+ cap_feats_len = -(-cap_feats_len // self.masked_pad_multiple) * self.masked_pad_multiple
+ cap_feats_len += offset
embeds += (None,)
freqs_cis += (None,)
else:
@@ -702,9 +704,17 @@ class NextDiT(nn.Module):
embeds += (siglip_feats,)
freqs_cis += (self.rope_embedder(siglip_pos_ids).movedim(1, 2),)
- B, C, H, W = x.shape
- x = self.x_embedder(x.view(B, C, H // pH, pH, W // pW, pW).permute(0, 2, 4, 3, 5, 1).flatten(3).flatten(1, 2))
- x_pos_ids = pos_ids_x(cap_feats_len + 1, H // pH, W // pW, bsz, device, transformer_options=transformer_options)
+ if x.ndim == 4:
+ x = x.unsqueeze(2)
+ B, C, F, H, W = x.shape
+ x = self.x_embedder(x.view(B, C, F, H // pH, pH, W // pW, pW).permute(0, 2, 3, 5, 4, 6, 1).flatten(4).flatten(1, 3))
+ x_pos_ids = torch.cat([pos_ids_x(cap_feats_len + 1 + f, H // pH, W // pW, bsz, device, transformer_options=transformer_options) for f in range(F)], dim=1)
+ for f, ref in enumerate(ref_frames): # Ming-Image: clean reference frames follow the target frames on the t axis
+ ref = comfy.ldm.common_dit.pad_to_patch_size(ref, (pH, pW))
+ rH, rW = ref.shape[-2], ref.shape[-1]
+ ref = self.x_embedder(ref.view(ref.shape[0], C, rH // pH, pH, rW // pW, pW).permute(0, 2, 4, 3, 5, 1).flatten(3).flatten(1, 2))
+ x = torch.cat((x, comfy.utils.repeat_to_batch_size(ref, B)), dim=1)
+ x_pos_ids = torch.cat((x_pos_ids, pos_ids_x(cap_feats_len + 1 + F + f, rH // pH, rW // pW, bsz, device, transformer_options=transformer_options)), dim=1)
if self.pad_tokens_multiple is not None:
x, pad_extra = pad_zimage(x, self.x_pad_token, self.pad_tokens_multiple)
x_pos_ids = torch.nn.functional.pad(x_pos_ids, (0, 0, 0, pad_extra))
@@ -715,7 +725,7 @@ class NextDiT(nn.Module):
def patchify_and_embed(
- self, x: torch.Tensor, cap_feats: torch.Tensor, cap_mask: torch.Tensor, t: torch.Tensor, num_tokens, ref_latents=[], ref_contexts=[], siglip_feats=[], transformer_options={}
+ self, x: torch.Tensor, cap_feats: torch.Tensor, cap_mask: torch.Tensor, t: torch.Tensor, num_tokens, ref_latents=[], ref_contexts=[], siglip_feats=[], direct_context=None, ref_frames=[], transformer_options={}
) -> Tuple[torch.Tensor, torch.Tensor, List[Tuple[int, int]], List[int], torch.Tensor]:
bsz = x.shape[0]
cap_mask = None # TODO?
@@ -749,7 +759,7 @@ class NextDiT(nn.Module):
H, W = x.shape[-2], x.shape[-1]
img_sizes = [(H, W)] * bsz
- out = self.embed_all(x, cap_feats, main_siglip, offset=start_t, omni=omni, transformer_options=transformer_options)
+ out = self.embed_all(x, cap_feats, main_siglip, offset=start_t, omni=omni, transformer_options=transformer_options, cap_extra=direct_context, ref_frames=ref_frames)
img_len = out[0][-1].shape[1]
cap_len = out[0][0].shape[1]
for i, e in enumerate(out[0]):
@@ -822,7 +832,7 @@ class NextDiT(nn.Module):
).execute(x, timesteps, context, num_tokens, attention_mask, **kwargs)
# def forward(self, x, t, cap_feats, cap_mask):
- def _forward(self, x, timesteps, context, num_tokens, attention_mask=None, ref_latents=[], ref_contexts=[], siglip_feats=[], transformer_options={}, **kwargs):
+ def _forward(self, x, timesteps, context, num_tokens, attention_mask=None, ref_latents=[], ref_contexts=[], siglip_feats=[], direct_context=None, ref_frames=[], transformer_options={}, **kwargs):
omni = len(ref_latents) > 0
if omni:
timesteps = torch.cat([timesteps * 0, timesteps], dim=0)
@@ -830,8 +840,9 @@ class NextDiT(nn.Module):
t = 1.0 - timesteps
cap_feats = context
cap_mask = attention_mask
- bs, c, h, w = x.shape
- x = comfy.ldm.common_dit.pad_to_patch_size(x, (self.patch_size, self.patch_size))
+ frames = x.shape[2] if x.ndim == 5 else None # Ming-Image latents carry a frame axis: the target, or composite + layers for Ming-Image-Layer
+ h, w = x.shape[-2], x.shape[-1]
+ x = comfy.ldm.common_dit.pad_to_patch_size(x, (1, self.patch_size, self.patch_size) if frames else (self.patch_size, self.patch_size))
"""
Forward pass of NextDiT.
t: (N,) tensor of diffusion timesteps
@@ -852,7 +863,7 @@ class NextDiT(nn.Module):
patches = transformer_options.get("patches", {})
x_is_tensor = isinstance(x, torch.Tensor)
- img, mask, img_size, cap_size, freqs_cis, timestep_zero_index = self.patchify_and_embed(x, cap_feats, cap_mask, adaln_input, num_tokens, ref_latents=ref_latents, ref_contexts=ref_contexts, siglip_feats=siglip_feats, transformer_options=transformer_options)
+ img, mask, img_size, cap_size, freqs_cis, timestep_zero_index = self.patchify_and_embed(x, cap_feats, cap_mask, adaln_input, num_tokens, ref_latents=ref_latents, ref_contexts=ref_contexts, siglip_feats=siglip_feats, direct_context=direct_context, ref_frames=ref_frames, transformer_options=transformer_options)
freqs_cis = freqs_cis.to(img.device)
transformer_options["total_blocks"] = len(self.layers)
@@ -870,7 +881,7 @@ class NextDiT(nn.Module):
img[:, :cap_size[0]] = out["txt"]
img = self.final_layer(img, adaln_input, timestep_zero_index=timestep_zero_index)
- img = self.unpatchify(img, img_size, cap_size, return_tensor=x_is_tensor)[:, :, :h, :w]
+ img = self.unpatchify(img, img_size, cap_size, return_tensor=x_is_tensor, frames=frames)[..., :h, :w]
return -img
diff --git a/comfy/model_base.py b/comfy/model_base.py
index f4d3ac838..4f731e7cd 100644
--- a/comfy/model_base.py
+++ b/comfy/model_base.py
@@ -1559,6 +1559,17 @@ class Lumina2(BaseModel):
out['ref_latents'] = list([1, 16, sum(map(lambda a: math.prod(a.size()[2:]), ref_latents))])
return out
+class MingImage(Lumina2):
+ def extra_conds(self, **kwargs):
+ ref_latents = kwargs.pop("reference_latents", None)
+ out = super().extra_conds(**kwargs)
+ direct_context = kwargs.get("direct_context", None)
+ if direct_context is not None:
+ out['direct_context'] = comfy.conds.CONDRegular(direct_context)
+ if ref_latents is not None:
+ out['ref_frames'] = comfy.conds.CONDList([self.process_latent_in(lat)[:, :, f] for lat in ref_latents for f in range(lat.shape[2])])
+ return out
+
class ZImagePixelSpace(Lumina2):
def __init__(self, model_config, model_type=ModelType.FLOW, device=None):
BaseModel.__init__(self, model_config, model_type, device=device, unet_model=comfy.ldm.lumina.model.NextDiTPixelSpace)
diff --git a/comfy/model_detection.py b/comfy/model_detection.py
index 1f2342647..6a1d288ec 100644
--- a/comfy/model_detection.py
+++ b/comfy/model_detection.py
@@ -609,13 +609,19 @@ def detect_unet_config(state_dict, key_prefix, metadata=None):
dit_config["allow_fp16"] = torch.std(state_dict['{}layers.{}.ffn_norm1.weight'.format(key_prefix, dit_config["n_layers"] - 2)], unbiased=False).item() < 0.42
except Exception:
pass
+ dec_cond_key = '{}dec_net.cond_embed.weight'.format(key_prefix)
if '{}cap_pad_token'.format(key_prefix) in state_dict_keys:
dit_config["pad_tokens_multiple"] = 32
+ # Ming-Image: Z-Image keys. Design ships without the learned pad tokens; Layer has them and only its repack metadata tells it apart.
+ ming_metadata = metadata is not None and "config" in metadata and json.loads(metadata["config"]).get("transformer", {}).get("image_model") == "ming_image"
+ if ming_metadata or ("pad_tokens_multiple" not in dit_config and dec_cond_key not in state_dict_keys):
+ dit_config["image_model"] = "ming_image"
+ if "pad_tokens_multiple" not in dit_config:
+ dit_config["masked_pad_multiple"] = 32
sig_weight = state_dict.get('{}siglip_embedder.0.weight'.format(key_prefix), None)
if sig_weight is not None:
dit_config["siglip_feat_dim"] = sig_weight.shape[0]
- dec_cond_key = '{}dec_net.cond_embed.weight'.format(key_prefix)
if dec_cond_key in state_dict_keys: # pixel-space variant
dit_config["image_model"] = "zimage_pixel"
# patch_size and in_channels are derived from x_embedder:
diff --git a/comfy/ops.py b/comfy/ops.py
index f4ac760c9..860e09ea0 100644
--- a/comfy/ops.py
+++ b/comfy/ops.py
@@ -19,6 +19,7 @@
import torch
import logging
import contextlib
+import dataclasses
import inspect
import comfy.model_management
from comfy.cli_args import args, PerformanceFeature
@@ -1549,8 +1550,18 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec
def _apply(self, fn, recurse=True):
return _quantized_apply(self, fn, recurse)
- def _load_from_state_dict(self, *args):
- _load_quantized_module(self, super()._load_from_state_dict, *args, load_extra_params=False)
+ def _load_from_state_dict(self, state_dict, prefix, *args):
+ layer_conf = state_dict.get(f"{prefix}comfy_quant", None)
+ quant_format = json.loads(layer_conf.numpy().tobytes()).get("format") if layer_conf is not None else None
+ scale = state_dict.get(f"{prefix}weight_scale", None)
+ if quant_format == "asym_w4a8_int8" or (quant_format == "int8_tensorwise" and scale is not None and scale.ndim == 3):
+ # per-row scaled layouts are 2-D only: keep the bank as [E * out, in] and slice rows per expert
+ for name in ("weight", "weight_scale", "weight_s_rel", "weight_s_channel"):
+ t = state_dict.get(f"{prefix}{name}", None)
+ if t is not None and t.ndim > 1 and t.shape[0] == self.num_experts:
+ state_dict[f"{prefix}{name}"] = t.reshape(self.num_experts * t.shape[1], *t.shape[2:])
+ self._orig_shape = (self.num_experts * self.out_features, self.in_features)
+ _load_quantized_module(self, super()._load_from_state_dict, state_dict, prefix, *args, load_extra_params=False)
def expert_weight(self, i: int):
"""Expert i's weight (Tensor or per-expert QuantizedTensor view)."""
@@ -1558,12 +1569,38 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec
return self._expert_qt_from(self.weight, i)
return self.weight[i]
+ def _cast_bank(self, input):
+ # A quantized bank stays quantized: the layouts dequantize one [out, in] matrix at a time, per expert.
+ if isinstance(self.weight, QuantizedTensor):
+ return CastBiasWeightContext(self, input=None, dtype=self.weight.dtype, device=input.device, bias_dtype=input.dtype, offloadable=True)
+ return CastBiasWeightContext(self, input, offloadable=True)
+
+ def _dequantize_bank(self, weight, dtype):
+ # in expert chunks: the W4A8 dequantize kernel allocates over twice its output in temporaries
+ flat = QuantizedTensor(weight._qdata, weight._layout_cls, dataclasses.replace(weight._params, orig_dtype=dtype))
+ out = torch.empty((self.num_experts, self.out_features, self.in_features), dtype=dtype, device=weight.device)
+ step = max(1, (256 << 20) // (self.out_features * self.in_features * out.element_size()))
+ for first in range(0, self.num_experts, step):
+ last = min(first + step, self.num_experts)
+ out[first:last] = self._bank_rows(flat, first, last).dequantize().view(last - first, self.out_features, self.in_features)
+ return out
+
+ def _bank_rows(self, weight: QuantizedTensor, first: int, last: int) -> QuantizedTensor:
+ """Experts [first, last) of a flat [E * out, in] bank as one QuantizedTensor."""
+ params = weight._params
+ rows = slice(first * self.out_features, last * self.out_features)
+ per_row = {f.name: getattr(params, f.name)[rows] for f in dataclasses.fields(params) if torch.is_tensor(getattr(params, f.name)) and getattr(params, f.name).ndim >= 1 and getattr(params, f.name).shape[0] == weight._qdata.shape[0]}
+ return QuantizedTensor(weight._qdata[rows], weight._layout_cls, dataclasses.replace(params, orig_shape=((last - first) * self.out_features, self.in_features), **per_row))
+
@contextlib.contextmanager
def bank_resident(self, input):
"""Cast the whole bank once; expert_linear inside reuses the cast.
Not re-entrant — do not nest calls on the same instance.
"""
- with CastBiasWeightContext(self, input, offloadable=True) as self._resident_bank:
+ with self._cast_bank(input) as (weight, bias):
+ if self._full_precision_mm and isinstance(weight, QuantizedTensor) and weight._qdata.ndim == 2: # flat per-row banks; 3-D banks dequantize per expert
+ weight = self._dequantize_bank(weight, input.dtype)
+ self._resident_bank = (weight, bias)
try:
yield self
finally:
@@ -1575,14 +1612,14 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec
if resident is not None:
weight, bias = resident
return self._expert_linear_impl(input, weight, bias, i)
- with CastBiasWeightContext(self, input, offloadable=True) as (weight, bias):
+ with self._cast_bank(input) as (weight, bias):
return self._expert_linear_impl(input, weight, bias, i)
def _expert_linear_impl(self, input, weight, bias, i):
if isinstance(weight, QuantizedTensor):
qw = self._expert_qt_from(weight, i)
else:
- qw = weight[i]
+ qw = cast_to_input(weight[i], input, copy=False)
b = cast_to_input(bias[i], input, copy=False) if bias is not None else None
if isinstance(qw, QuantizedTensor):
@@ -1594,12 +1631,13 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec
if use_fast:
qin = QuantizedTensor.from_float(input, self.layout_type)
return torch.nn.functional.linear(qin, qw, b)
- out = input @ qw.dequantize().t()
- return out + b if b is not None else out
+ qw = cast_to_input(qw.dequantize(), input, copy=False)
return torch.nn.functional.linear(input, qw, b)
def _expert_qt_from(self, weight: QuantizedTensor, i: int) -> QuantizedTensor:
"""Build a per-expert QuantizedTensor by indexing into a resident bank."""
+ if weight._qdata.ndim == 2:
+ return self._bank_rows(weight, i, i + 1)
params = weight._params
kwargs = {
"scale": params.scale[i] if params.scale.dim() else params.scale,
@@ -1610,6 +1648,8 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec
kwargs["block_scale"] = params.block_scale[i]
if hasattr(params, "quant_group_size"):
kwargs["quant_group_size"] = params.quant_group_size
+ if hasattr(params, "convrot"):
+ kwargs["convrot"] = params.convrot
if hasattr(params, "convrot_groupsize"):
kwargs["convrot_groupsize"] = params.convrot_groupsize
if hasattr(params, "linear_dtype"):
diff --git a/comfy/sd.py b/comfy/sd.py
index ea42f6a47..104caae9a 100644
--- a/comfy/sd.py
+++ b/comfy/sd.py
@@ -67,6 +67,7 @@ import comfy.text_encoders.qwen_image
import comfy.text_encoders.qwen_image21
import comfy.text_encoders.hunyuan_image
import comfy.text_encoders.z_image
+import comfy.text_encoders.ming_image
import comfy.text_encoders.krea2
import comfy.text_encoders.mage_flow
import comfy.text_encoders.ideogram4
@@ -1640,9 +1641,12 @@ class TEModel(Enum):
QWEN3VL_8B = 35
GEMMA_4_12B = 36
QWEN3VL_32B = 37
+ MING_IMAGE = 38
def detect_te_model(sd):
+ if "thinker.layers.1.mlp.image_gate.proj.weight" in sd:
+ return TEModel.MING_IMAGE
if "text_model.encoder.layers.30.mlp.fc1.weight" in sd:
return TEModel.CLIP_G
if "text_model.encoder.layers.22.mlp.fc1.weight" in sd:
@@ -1902,6 +1906,11 @@ def load_text_encoder_state_dicts(state_dicts=[], embedding_directory=None, clip
clip_target.clip = comfy.text_encoders.flux.flux2_te(**llama_detect(clip_data), pruned=te_model == TEModel.MISTRAL3_24B_PRUNED_FLUX2)
clip_target.tokenizer = comfy.text_encoders.flux.Flux2Tokenizer
tokenizer_data["tekken_model"] = clip_data[0].get("tekken_model", None)
+ elif te_model == TEModel.MING_IMAGE:
+ tokenizer_data["tokenizer_json"] = clip_data[0].get("tokenizer_json", None)
+ quant = comfy.utils.detect_layer_quantization(clip_data[0], "")
+ clip_target.clip = comfy.text_encoders.ming_image.te(dtype_llama=clip_data[0]["thinker.norm.weight"].dtype, llama_quantization_metadata=quant)
+ clip_target.tokenizer = comfy.text_encoders.ming_image.MingImageTokenizer
elif te_model == TEModel.GPT_OSS_20B:
clip_target.clip = comfy.text_encoders.gpt_oss.lens_te(**llama_detect(clip_data))
clip_target.tokenizer = comfy.text_encoders.gpt_oss.LensTokenizer
@@ -2346,7 +2355,7 @@ def load_diffusion_model_state_dict(sd, model_options={}, metadata=None, disable
else:
new_sd = model_detection.convert_diffusers_mmdit(sd, "")
if new_sd is not None: #diffusers mmdit
- model_config = model_detection.model_config_from_unet(new_sd, "")
+ model_config = model_detection.model_config_from_unet(new_sd, "", metadata=metadata)
if model_config is None:
return None
else: #diffusers unet
diff --git a/comfy/supported_models.py b/comfy/supported_models.py
index b5cc18736..b4ca22715 100644
--- a/comfy/supported_models.py
+++ b/comfy/supported_models.py
@@ -27,6 +27,7 @@ import comfy.text_encoders.qwen_image21
import comfy.text_encoders.hunyuan_image
import comfy.text_encoders.kandinsky5
import comfy.text_encoders.z_image
+import comfy.text_encoders.ming_image
import comfy.text_encoders.ideogram4
import comfy.text_encoders.boogu
import comfy.text_encoders.krea2
@@ -1242,6 +1243,24 @@ class ZImagePixelSpace(ZImage):
def get_model(self, state_dict, prefix="", device=None):
return model_base.ZImagePixelSpace(self, device=device)
+class MingImage(ZImage):
+ unet_config = {
+ "image_model": "ming_image",
+ }
+
+ sampling_settings = {
+ "multiplier": 1.0,
+ "shift": 3.16, # reference dynamic shift at the 1024 bucket
+ }
+
+ latent_format = latent_formats.MingImage
+
+ def get_model(self, state_dict, prefix="", device=None):
+ return model_base.MingImage(self, device=device)
+
+ def clip_target(self, state_dict={}):
+ return supported_models_base.ClipTarget(comfy.text_encoders.ming_image.MingImageTokenizer, comfy.text_encoders.ming_image.te())
+
class PixelDiTT2I(supported_models_base.BASE):
unet_config = {
"image_model": "pixeldit_t2i",
@@ -2602,6 +2621,7 @@ models = [
CosmosT2IPredict2,
CosmosI2VPredict2,
ZImagePixelSpace,
+ MingImage,
ZImage,
PiD,
PixelDiTT2I,
diff --git a/comfy/text_encoders/gpt_oss.py b/comfy/text_encoders/gpt_oss.py
index 066796b6a..744ea6c92 100644
--- a/comfy/text_encoders/gpt_oss.py
+++ b/comfy/text_encoders/gpt_oss.py
@@ -13,7 +13,7 @@ import torch.nn.functional as F
import comfy.ops
from comfy import sd1_clip
from comfy.ldm.modules.attention import optimized_attention_for_device
-from comfy.text_encoders.llama import RMSNorm, apply_rope
+from comfy.text_encoders.llama import RMSNorm, apply_rope, moe_experts_forward
@dataclass
@@ -205,32 +205,7 @@ class GptOssExperts(nn.Module):
return torch.addcmul(glu, up, glu)
def forward(self, hidden_states: torch.Tensor, router_indices: torch.Tensor, routing_weights: torch.Tensor) -> torch.Tensor:
- N = hidden_states.shape[0]
- top_k = router_indices.shape[-1]
- H = hidden_states.shape[-1]
-
- per_pair = torch.zeros((N * top_k, H), dtype=hidden_states.dtype, device=hidden_states.device)
-
- expert_mask = F.one_hot(router_indices, num_classes=self.num_experts).permute(2, 1, 0)
- expert_hit = torch.greater(expert_mask.sum(dim=(-1, -2)), 0).nonzero()
-
- with self.gate_up_proj.bank_resident(hidden_states) as gate_up_bank, \
- self.down_proj.bank_resident(hidden_states) as down_bank:
- for ei in expert_hit:
- expert_idx = int(ei.item())
- top_k_pos, token_idx = torch.where(expert_mask[expert_idx])
- current = hidden_states[token_idx]
-
- gate_up = gate_up_bank.expert_linear(current, expert_idx)
- gated = self._apply_gate(gate_up)
- expert_out = down_bank.expert_linear(gated, expert_idx)
-
- weighted = expert_out * routing_weights[token_idx, top_k_pos, None]
-
- flat_idx = token_idx * top_k + top_k_pos
- per_pair[flat_idx] = weighted.to(per_pair.dtype)
-
- return per_pair.view(N, top_k, H).sum(dim=1)
+ return moe_experts_forward(hidden_states, router_indices, routing_weights, self.num_experts, self.gate_up_proj, self.down_proj, self._apply_gate)
class GptOssMLP(nn.Module):
diff --git a/comfy/text_encoders/llama.py b/comfy/text_encoders/llama.py
index 96577011a..ba55fbf14 100644
--- a/comfy/text_encoders/llama.py
+++ b/comfy/text_encoders/llama.py
@@ -558,6 +558,29 @@ def precompute_freqs_cis(head_dim, position_ids, theta, rope_scale=None, rope_di
return out
+def moe_experts_forward(x, topk_idx, topk_weight, num_experts, gate_up_proj, down_proj, activation):
+ num_tokens, top_k = topk_idx.shape
+ # group the (token, slot) assignments by expert: one host sync per call instead of two per expert
+ order = torch.argsort(topk_idx.reshape(-1))
+ counts = torch.bincount(topk_idx.reshape(-1), minlength=num_experts).tolist()
+ sorted_x = x[order // top_k]
+ weight = topk_weight.reshape(-1)[order].unsqueeze(1)
+ sorted_out = torch.empty_like(sorted_x)
+
+ start = 0
+ with gate_up_proj.bank_resident(x) as gate_up_bank, down_proj.bank_resident(x) as down_bank:
+ for expert_idx, n in enumerate(counts):
+ if n == 0:
+ continue
+ gated = activation(gate_up_bank.expert_linear(sorted_x[start:start + n], expert_idx))
+ sorted_out[start:start + n] = (down_bank.expert_linear(gated, expert_idx) * weight[start:start + n]).to(sorted_out.dtype)
+ start += n
+
+ out = torch.empty_like(sorted_out)
+ out[order] = sorted_out
+ return out.view(num_tokens, top_k, -1).sum(dim=1)
+
+
def rope_matrix(freqs_cis):
if torch.is_tensor(freqs_cis):
return freqs_cis
diff --git a/comfy/text_encoders/ming_image.py b/comfy/text_encoders/ming_image.py
new file mode 100644
index 000000000..cf85fee6f
--- /dev/null
+++ b/comfy/text_encoders/ming_image.py
@@ -0,0 +1,349 @@
+from dataclasses import dataclass
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from tokenizers import Tokenizer
+
+import comfy.ops
+from comfy import sd1_clip
+from comfy.ldm.modules.attention import optimized_attention_for_device
+from comfy.text_encoders.llama import MLP, RMSNorm, TransformerBlock, moe_experts_forward, precompute_freqs_cis
+from comfy.text_encoders.qwen35 import apply_partial_rope
+from comfy.text_encoders import qwen_vl
+
+IMAGE_PATCH_TOKEN = 157157
+IMAGE_BLOCK = "<image><imagePatch></image>"
+
+
+@dataclass
+class BailingMoeV2Config:
+ vocab_size: int = 157184
+ hidden_size: int = 2048
+ intermediate_size: int = 5120
+ moe_intermediate_size: int = 512
+ num_hidden_layers: int = 20
+ num_attention_heads: int = 16
+ num_key_value_heads: int = 4
+ head_dim: int = 128
+ rope_dim: int = 64
+ rope_theta: float = 600000.0
+ rms_norm_eps: float = 1e-6
+ num_experts: int = 256
+ num_experts_per_tok: int = 8
+ n_group: int = 8
+ topk_group: int = 4
+ routed_scaling_factor: float = 2.5
+ mlp_activation: str = "silu"
+
+
+@dataclass
+class MingConnectorConfig:
+ hidden_size: int = 1536
+ intermediate_size: int = 8960
+ num_hidden_layers: int = 28
+ num_attention_heads: int = 12
+ num_key_value_heads: int = 2
+ head_dim: int = 128
+ rms_norm_eps: float = 1e-6
+ rope_theta: float = 1000000.0
+ qkv_bias: bool = True
+ rms_norm_add: bool = False
+ mlp_activation: str = "silu"
+ q_norm = None
+ k_norm = None
+
+
+class BailingAttention(nn.Module):
+ def __init__(self, config, device=None, dtype=None, ops=None):
+ super().__init__()
+ self.num_heads = config.num_attention_heads
+ self.num_kv_heads = config.num_key_value_heads
+ self.head_dim = config.head_dim
+ self.rope_dim = config.rope_dim
+ self.query_key_value = ops.Linear(config.hidden_size, (self.num_heads + 2 * self.num_kv_heads) * self.head_dim, bias=False, device=device, dtype=dtype)
+ self.q_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps, device=device, dtype=dtype)
+ self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps, device=device, dtype=dtype)
+ self.dense = ops.Linear(self.num_heads * self.head_dim, config.hidden_size, bias=False, device=device, dtype=dtype)
+
+ def forward(self, x, attention_mask, freqs_cis, optimized_attention):
+ batch, seq_len, _ = x.shape
+ qkv = self.query_key_value(x).view(batch, seq_len, -1, self.head_dim).transpose(1, 2)
+ xq, xk, xv = qkv.split((self.num_heads, self.num_kv_heads, self.num_kv_heads), dim=1)
+ xq = self.q_norm(xq)
+ xk = self.k_norm(xk)
+ xq, xk = apply_partial_rope(xq, xk, freqs_cis, self.rope_dim)
+ out = optimized_attention(xq, xk, xv, self.num_heads, mask=attention_mask, skip_reshape=True, enable_gqa=True)
+ return self.dense(out)
+
+
+class BailingGate(nn.Module):
+ def __init__(self, config, device=None, dtype=None, ops=None):
+ super().__init__()
+ self.top_k = config.num_experts_per_tok
+ self.n_group = config.n_group
+ self.topk_group = config.topk_group
+ self.routed_scaling_factor = config.routed_scaling_factor
+ self.proj = ops.Linear(config.hidden_size, config.num_experts, bias=False, device=device, dtype=dtype)
+ self.expert_bias = nn.Parameter(torch.empty(config.num_experts, device=device, dtype=torch.float32))
+
+ def forward(self, x):
+ scores = torch.sigmoid(self.proj(x.float()))
+ routing = scores + comfy.ops.cast_to_input(self.expert_bias, scores, copy=False)
+ num_tokens = routing.shape[0]
+ grouped = routing.view(num_tokens, self.n_group, -1)
+ group_idx = torch.topk(grouped.topk(2, dim=-1)[0].sum(dim=-1), k=self.topk_group, dim=-1, sorted=False)[1]
+ group_mask = torch.zeros(num_tokens, self.n_group, dtype=torch.bool, device=x.device).scatter_(1, group_idx, True)
+ topk_idx = torch.topk(grouped.masked_fill(~group_mask.unsqueeze(-1), float("-inf")).view(num_tokens, -1), k=self.top_k, dim=-1, sorted=False)[1]
+ topk_weight = torch.gather(scores, dim=1, index=topk_idx)
+ return topk_idx, topk_weight / (topk_weight.sum(dim=-1, keepdim=True) + 1e-20) * self.routed_scaling_factor
+
+
+class BailingExperts(nn.Module):
+ def __init__(self, config, device=None, dtype=None, ops=None):
+ super().__init__()
+ self.num_experts = config.num_experts
+ self.gate_up_proj = ops.MoEExperts(num_experts=config.num_experts, in_features=config.hidden_size, out_features=2 * config.moe_intermediate_size, bias=False, device=device, dtype=dtype)
+ self.down_proj = ops.MoEExperts(num_experts=config.num_experts, in_features=config.moe_intermediate_size, out_features=config.hidden_size, bias=False, device=device, dtype=dtype)
+
+ def forward(self, x, topk_idx, topk_weight):
+ return moe_experts_forward(x, topk_idx, topk_weight, self.num_experts, self.gate_up_proj, self.down_proj, comfy.ops._swiglu_eager)
+
+
+class BailingSparseMoe(nn.Module):
+ def __init__(self, config, device=None, dtype=None, ops=None):
+ super().__init__()
+ self.gate = BailingGate(config, device=device, dtype=dtype, ops=ops)
+ self.image_gate = BailingGate(config, device=device, dtype=dtype, ops=ops)
+ self.experts = BailingExperts(config, device=device, dtype=dtype, ops=ops)
+ self.shared_experts = MLP(config, device=device, dtype=dtype, ops=ops, intermediate_size=config.moe_intermediate_size)
+
+ def forward(self, x, image_mask):
+ batch, seq_len, hidden = x.shape
+ flat = x.reshape(-1, hidden)
+ text_idx, text_weight = self.gate(flat)
+ image_idx, image_weight = self.image_gate(flat)
+ mask = image_mask.reshape(-1, 1)
+ out = self.experts(flat, torch.where(mask, image_idx, text_idx), torch.where(mask, image_weight, text_weight))
+ return out.view(batch, seq_len, hidden) + self.shared_experts(x)
+
+
+class BailingDecoderLayer(nn.Module):
+ def __init__(self, config, index, device=None, dtype=None, ops=None):
+ super().__init__()
+ self.attention = BailingAttention(config, device=device, dtype=dtype, ops=ops)
+ self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps, device=device, dtype=dtype)
+ self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps, device=device, dtype=dtype)
+ self.dense = index == 0
+ if self.dense:
+ self.mlp = MLP(config, device=device, dtype=dtype, ops=ops)
+ else:
+ self.mlp = BailingSparseMoe(config, device=device, dtype=dtype, ops=ops)
+
+ def forward(self, x, attention_mask, freqs_cis, optimized_attention, image_mask):
+ x = x + self.attention(self.input_layernorm(x), attention_mask, freqs_cis, optimized_attention)
+ h = self.post_attention_layernorm(x)
+ if self.dense:
+ return x + self.mlp(h)
+ return x + self.mlp(h, image_mask)
+
+
+class BailingMoeV2(nn.Module):
+ def __init__(self, config, device=None, dtype=None, ops=None):
+ super().__init__()
+ self.config = config
+ self.embed_tokens = ops.Embedding(config.vocab_size, config.hidden_size, device=device, dtype=dtype)
+ self.layers = nn.ModuleList([BailingDecoderLayer(config, i, device=device, dtype=dtype, ops=ops) for i in range(config.num_hidden_layers)])
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps, device=device, dtype=dtype)
+
+ def freqs_cis(self, seq_len, blocks, device):
+ # video_rope: an image block takes one text position, its tokens form an (h, w) grid centered there
+ t = torch.arange(seq_len, device=device, dtype=torch.float32)
+ for start, gh, gw in blocks:
+ t[start + gh * gw:] -= gh * gw - 1
+ h = t.clone()
+ w = t.clone()
+ shift = 0
+ for start, gh, gw in blocks:
+ end = start + gh * gw
+ t[start:end] = start - shift
+ h[start:end] = start - shift + (torch.arange(gh, device=device) - (gh - 1) // 2).repeat_interleave(gw)
+ w[start:end] = start - shift + (torch.arange(gw, device=device) - (gw - 1) // 2).repeat(gh)
+ shift += gh * gw - 1
+ inv_freq = 1.0 / (self.config.rope_theta ** (torch.arange(0, self.config.rope_dim, 2, device=device, dtype=torch.float32) / self.config.rope_dim))
+ freqs = t[:, None] * inv_freq
+ freqs[:, 0:24:2] = h[:, None] * inv_freq[0:24:2]
+ freqs[:, 1:24:2] = w[:, None] * inv_freq[1:24:2]
+ freqs = freqs.unsqueeze(0)
+ return torch.cat((freqs, freqs), dim=-1).cos(), freqs.sin(), -freqs.sin()
+
+ def forward(self, x, attention_mask, blocks, capture_layers):
+ seq_len = x.shape[1]
+ freqs_cis = self.freqs_cis(seq_len, blocks, x.device)
+ neg = torch.finfo(x.dtype).min / 4
+ mask = torch.full((seq_len, seq_len), neg, dtype=x.dtype, device=x.device).triu_(1)
+ mask = mask + (attention_mask == 0).to(x.dtype)[:, None, None, :] * neg
+ optimized_attention = optimized_attention_for_device(x.device, mask=True, small_input=True)
+ image_mask = torch.zeros(x.shape[:2], dtype=torch.bool, device=x.device)
+ for start, gh, gw in blocks:
+ image_mask[:, start:start + gh * gw] = True
+
+ captured = []
+ for i, layer in enumerate(self.layers):
+ if i in capture_layers:
+ captured.append(x)
+ x = layer(x, mask, freqs_cis, optimized_attention, image_mask)
+ x = self.norm(x)
+ captured.append(x)
+ return x, captured
+
+
+class MingConnector(nn.Module):
+ def __init__(self, config, device=None, dtype=None, ops=None):
+ super().__init__()
+ self.config = config
+ self.layers = nn.ModuleList([TransformerBlock(config, index=i, device=device, dtype=dtype, ops=ops) for i in range(config.num_hidden_layers)])
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps, device=device, dtype=dtype)
+
+ def forward(self, x):
+ position_ids = torch.arange(x.shape[1], device=x.device).unsqueeze(0)
+ freqs_cis = precompute_freqs_cis(self.config.head_dim, position_ids, self.config.rope_theta, device=x.device)
+ optimized_attention = optimized_attention_for_device(x.device, mask=False, small_input=True)
+ for layer in self.layers:
+ x, _ = layer(x, attention_mask=None, freqs_cis=freqs_cis, optimized_attention=optimized_attention)
+ return self.norm(x)
+
+
+class MingImageEncoder(nn.Module):
+ capture_layers = (5, 12)
+
+ def __init__(self, config_dict, dtype, device, operations):
+ super().__init__()
+ config = BailingMoeV2Config(**config_dict)
+ connector_config = MingConnectorConfig()
+ self.num_layers = config.num_hidden_layers
+ self.thinker = BailingMoeV2(config, device=device, dtype=dtype, ops=operations)
+ self.vision = qwen_vl.Qwen2VLVisionTransformer(hidden_size=1280, output_hidden_size=8192, intermediate_size=3456, num_heads=16, num_layers=32, device=device, dtype=dtype, ops=operations)
+ self.linear_proj = nn.Sequential(
+ operations.Linear(8192, config.hidden_size, device=device, dtype=dtype),
+ nn.GELU(),
+ operations.Linear(config.hidden_size, config.hidden_size, device=device, dtype=dtype),
+ )
+ self.connector = MingConnector(connector_config, device=device, dtype=dtype, ops=operations)
+ self.query_tokens = nn.Parameter(torch.empty(256, config.hidden_size, device=device, dtype=dtype))
+ self.proj_in = operations.Linear(config.hidden_size, connector_config.hidden_size, device=device, dtype=dtype)
+ self.proj_out = operations.Linear(connector_config.hidden_size, 2560, device=device, dtype=dtype)
+ direct_dim = config.hidden_size * (len(self.capture_layers) + 1)
+ self.proj_directvlm = nn.Sequential(
+ operations.RMSNorm(direct_dim, eps=1e-5, elementwise_affine=True, device=device, dtype=dtype),
+ operations.Linear(direct_dim, 3840, device=device, dtype=dtype),
+ )
+
+ def get_input_embeddings(self):
+ return self.thinker.embed_tokens
+
+ def preprocess_embed(self, embed, device):
+ if embed["type"] == "query":
+ return self.query_tokens, None
+ if embed["type"] == "image":
+ pixels, grid = qwen_vl.process_qwen2vl_images(embed["data"], min_pixels=451584, max_pixels=451584)
+ feats = self.vision(pixels.to(device, dtype=torch.float32), grid)
+ return F.normalize(self.linear_proj(feats), dim=-1), grid
+ return None, None
+
+ def forward(self, embeds, attention_mask, embeds_info):
+ blocks = [(e["index"], int(e["extra"][0][1]) // 2, int(e["extra"][0][2]) // 2) if e["type"] == "image" else (e["index"], 1, e["size"]) for e in embeds_info]
+ start, _, size = blocks[-1]
+ hidden, captured = self.thinker(embeds, attention_mask, blocks, self.capture_layers)
+ cap_feats = self.proj_out(self.connector(self.proj_in(hidden[:, start:start + size])))
+ direct = torch.cat([c[:, :start - 1] for c in captured], dim=-1)
+ return cap_feats, self.proj_directvlm(direct)
+
+
+class _MingRawTokenizer:
+ def __init__(self, tokenizer_json_bytes=None, **kwargs):
+ if isinstance(tokenizer_json_bytes, torch.Tensor):
+ tokenizer_json_bytes = tokenizer_json_bytes.numpy().tobytes()
+ if not isinstance(tokenizer_json_bytes, bytes):
+ raise ValueError("The Ming-Image text encoder file must contain the tokenizer_json tensor.")
+ self.tokenizer = Tokenizer.from_str(tokenizer_json_bytes.decode("utf-8"))
+
+ @classmethod
+ def from_pretrained(cls, tokenizer_data, **kwargs):
+ return cls(tokenizer_json_bytes=tokenizer_data, **kwargs)
+
+ def __call__(self, text):
+ return {"input_ids": self.tokenizer.encode(text, add_special_tokens=False).ids}
+
+ def get_vocab(self):
+ return self.tokenizer.get_vocab()
+
+ def decode(self, ids, **kwargs):
+ return self.tokenizer.decode(ids, skip_special_tokens=kwargs.get("skip_special_tokens", False))
+
+
+class MingTokenizer(sd1_clip.SDTokenizer):
+ def __init__(self, embedding_directory=None, tokenizer_data={}):
+ self.tokenizer_json_data = tokenizer_data.get("tokenizer_json", None)
+ super().__init__(self.tokenizer_json_data, pad_with_end=False, embedding_directory=embedding_directory, embedding_size=2048, embedding_key='ming_image', tokenizer_class=_MingRawTokenizer, has_start_token=False, has_end_token=False, pad_to_max_length=False, max_length=99999999, min_length=1, pad_token=156892, disable_weights=True, tokenizer_data=tokenizer_data)
+
+ def state_dict(self):
+ return {"tokenizer_json": self.tokenizer_json_data}
+
+
+class MingImageTokenizer(sd1_clip.SD1Tokenizer):
+ def __init__(self, embedding_directory=None, tokenizer_data={}):
+ super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, name="ming_image", tokenizer=MingTokenizer)
+ self.llama_template = "<role>SYSTEM</role>你是一个友好的AI助手。\n\ndetailed thinking off<|role_end|><role>HUMAN</role>{}<|role_end|><role>ASSISTANT</role>" + IMAGE_BLOCK
+
+ def tokenize_with_weights(self, text, return_word_ids=False, llama_template=None, images=[], **kwargs):
+ if llama_template is None:
+ llama_template = self.llama_template
+ if len(images) > 0: # reference images lead the user turn, separated by blank lines like the vendor processor
+ llama_template = llama_template.replace("{}", "\n\n".join([IMAGE_BLOCK] * len(images)) + "\n{}")
+ tokens = super().tokenize_with_weights(llama_template.format(text), return_word_ids=return_word_ids, **kwargs)
+ images = iter(images)
+ for r in tokens["ming_image"]:
+ for i in range(len(r)):
+ if r[i][0] == IMAGE_PATCH_TOKEN:
+ image = next(images, None)
+ r[i] = ({"type": "query"} if image is None else {"type": "image", "data": image},) + r[i][1:]
+ return tokens
+
+
+class MingImageClipModel(sd1_clip.SDClipModel):
+ def __init__(self, device="cpu", dtype=None, model_options={}):
+ super().__init__(device=device, layer="last", layer_idx=None, textmodel_json_config={}, dtype=dtype, special_tokens={"pad": 156892}, layer_norm_hidden_state=False, model_class=MingImageEncoder, enable_attention_masks=True, return_attention_masks=False, model_options=model_options)
+
+ def forward(self, tokens):
+ if self.execution_device is None:
+ device = self.transformer.get_input_embeddings().weight.device
+ else:
+ device = self.execution_device
+ embeds, attention_mask, num_tokens, embeds_info = self.process_tokens(tokens, device)
+ cap_feats, direct_context = self.transformer(embeds, attention_mask, embeds_info)
+ return cap_feats.float(), None, {"direct_context": direct_context.float()}
+
+
+class MingImageTEModel(sd1_clip.SD1ClipModel):
+ def __init__(self, device="cpu", dtype=None, model_options={}):
+ super().__init__(device=device, dtype=dtype, name="ming_image", clip_model=MingImageClipModel, model_options=model_options)
+
+ def memory_estimation_function(self, tokens, device=None):
+ # both expert banks of a MoE layer sit in fp32 while it runs, on top of the weights; images expand to a few hundred tokens each
+ config = BailingMoeV2Config()
+ banks = 3 * config.hidden_size * config.moe_intermediate_size * config.num_experts * 4
+ num_tokens = sum(600 if isinstance(t[0], dict) else 1 for batch in tokens.get("ming_image", []) for t in batch)
+ return banks + num_tokens * config.hidden_size * 64
+
+
+def te(dtype_llama=None, llama_quantization_metadata=None):
+ class MingImageTEModel_(MingImageTEModel):
+ def __init__(self, device="cpu", dtype=None, model_options={}):
+ if dtype_llama is not None:
+ dtype = dtype_llama
+ model_options = model_options.copy()
+ if "custom_operations" not in model_options:
+ model_options["custom_operations"] = comfy.ops.mixed_precision_ops(llama_quantization_metadata or {}, dtype, full_precision_mm=True)
+ super().__init__(device=device, dtype=dtype, model_options=model_options)
+ return MingImageTEModel_
diff --git a/comfy_extras/nodes_ming.py b/comfy_extras/nodes_ming.py
new file mode 100644
index 000000000..6dce384ed
--- /dev/null
+++ b/comfy_extras/nodes_ming.py
@@ -0,0 +1,54 @@
+import comfy.utils
+import node_helpers
+from typing_extensions import override
+from comfy_api.latest import ComfyExtension, io
+
+
+class TextEncodeMingImageEdit(io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="TextEncodeMingImageEdit",
+ display_name="Text Encode Ming Image Edit",
+ category="model/conditioning/ming image",
+ inputs=[
+ io.Clip.Input("clip"),
+ io.Vae.Input("vae", optional=True, tooltip="Without a VAE the images only condition the text encoder through the vision tower."),
+ io.String.Input("prompt", multiline=True, dynamic_prompts=True),
+ io.Autogrow.Input(
+ "images",
+ template=io.Autogrow.TemplateNames(
+ io.Image.Input("image"),
+ names=[f"image_{i}" for i in range(1, 9)],
+ min=0,
+ ),
+ tooltip="Optional reference images, seen by the text encoder and appended to the latent sequence as clean frames. Later images are resized to the first one, and the sampled latent should match its size.",
+ ),
+ ],
+ outputs=[
+ io.Conditioning.Output(),
+ ],
+ )
+
+ @classmethod
+ def execute(cls, clip, prompt, images: io.Autogrow.Type, vae=None) -> io.NodeOutput:
+ images = [images[name] for name in sorted(images, key=lambda n: int(n.rsplit("_", 1)[-1])) if images[name] is not None]
+ tokens = clip.tokenize(prompt, images=[image[:, :, :, :3] for image in images])
+ conditioning = clip.encode_from_tokens_scheduled(tokens)
+ if vae is not None and len(images) > 0:
+ height, width = images[0].shape[1:3] # every reference frame shares the canvas of the first one, as the vendor processor does
+ ref_latents = [vae.encode(comfy.utils.common_upscale(image.movedim(-1, 1), width, height, "bilinear", "disabled").movedim(1, -1)) for image in images]
+ conditioning = node_helpers.conditioning_set_values(conditioning, {"reference_latents": ref_latents}, append=True)
+ return io.NodeOutput(conditioning)
+
+
+class MingExtension(ComfyExtension):
+ @override
+ async def get_node_list(self) -> list[type[io.ComfyNode]]:
+ return [
+ TextEncodeMingImageEdit,
+ ]
+
+
+async def comfy_entrypoint() -> MingExtension:
+ return MingExtension()
diff --git a/nodes.py b/nodes.py
index 9bbba5a29..1d3fd6b16 100644
--- a/nodes.py
+++ b/nodes.py
@@ -293,6 +293,9 @@ class ConditioningZeroOut:
conditioning_scale = d.get("conditioning_scale", None)
if conditioning_scale is not None:
d["conditioning_scale"] = torch.zeros_like(conditioning_scale)
+ direct_context = d.get("direct_context", None)
+ if direct_context is not None:
+ d["direct_context"] = torch.zeros_like(direct_context)
n = [torch.zeros_like(t[0]), d]
c.append(n)
return (c, )
@@ -2487,6 +2490,7 @@ async def init_builtin_extra_nodes():
"nodes_seedvr.py",
"nodes_context_windows.py",
"nodes_qwen.py",
+ "nodes_ming.py",
"nodes_mage.py",
"nodes_joyimage.py",
"nodes_boogu.py",