Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Fix saving/loading ming image model breaking detection. (#16514)

comfyanonymous41c83ee79f46
diff --git a/comfy/ldm/lumina/model.py b/comfy/ldm/lumina/model.py
index 62ccdad11..bc9528875 100644
--- a/comfy/ldm/lumina/model.py
+++ b/comfy/ldm/lumina/model.py
@@ -479,6 +479,9 @@ class NextDiT(nn.Module):
self.pad_tokens_multiple = pad_tokens_multiple
self.masked_pad_multiple = masked_pad_multiple
+ if image_model == "ming_image":
+ self.register_buffer("__ming_image__", torch.empty(0))
+
self.x_embedder = operation_settings.get("operations").Linear(
in_features=patch_size * patch_size * in_channels,
out_features=dim,
diff --git a/comfy/model_detection.py b/comfy/model_detection.py
index 6a1d288ec..46332edff 100644
--- a/comfy/model_detection.py
+++ b/comfy/model_detection.py
@@ -612,9 +612,9 @@ def detect_unet_config(state_dict, key_prefix, metadata=None):
dec_cond_key = '{}dec_net.cond_embed.weight'.format(key_prefix)
if '{}cap_pad_token'.format(key_prefix) in state_dict_keys:
dit_config["pad_tokens_multiple"] = 32
- # Ming-Image: Z-Image keys. Design ships without the learned pad tokens; Layer has them and only its repack metadata tells it apart.
+ # Ming-Image Design has no learned pad tokens; Layer needs a saved marker or repack metadata.
ming_metadata = metadata is not None and "config" in metadata and json.loads(metadata["config"]).get("transformer", {}).get("image_model") == "ming_image"
- if ming_metadata or ("pad_tokens_multiple" not in dit_config and dec_cond_key not in state_dict_keys):
+ if '{}__ming_image__'.format(key_prefix) in state_dict_keys or ming_metadata or ("pad_tokens_multiple" not in dit_config and dec_cond_key not in state_dict_keys):
dit_config["image_model"] = "ming_image"
if "pad_tokens_multiple" not in dit_config:
dit_config["masked_pad_multiple"] = 32
diff --git a/tests-unit/comfy_test/model_detection_test.py b/tests-unit/comfy_test/model_detection_test.py
index 94da2fdf6..de4fe8e09 100644
--- a/tests-unit/comfy_test/model_detection_test.py
+++ b/tests-unit/comfy_test/model_detection_test.py
@@ -3,6 +3,8 @@ from collections import defaultdict
import torch
from comfy.model_detection import detect_unet_config, model_config_from_unet, model_config_from_unet_config
+from comfy.ldm.lumina.model import NextDiT
+import comfy.ops
import comfy.supported_models
@@ -301,6 +303,37 @@ class TestModelDetection:
assert unet_config["proj_in_channels_shape"] == 2048
assert unet_config["proj_in_channels_structure"] == 1024
+ def test_ming_image_save_preserves_identity_without_metadata(self):
+ for learned_padding in (False, True):
+ sd = {
+ "cap_embedder.1.weight": torch.empty(3840, 2560, device="meta"),
+ "noise_refiner.0.attention.k_norm.weight": torch.empty(128, device="meta"),
+ }
+ if learned_padding:
+ sd["cap_pad_token"] = torch.empty(1, 3840, device="meta")
+ sd["x_pad_token"] = torch.empty(1, 3840, device="meta")
+ assert type(model_config_from_unet(sd, "")) is comfy.supported_models.ZImage
+
+ model_config = model_config_from_unet(sd, "", metadata={"config": '{"transformer": {"image_model": "ming_image"}}'})
+ model = NextDiT(**model_config.unet_config, device="meta", operations=comfy.ops.manual_cast)
+ original_sd = model.state_dict()
+ del original_sd["__ming_image__"]
+ missing, unexpected = model.load_state_dict(original_sd, strict=False, assign=True)
+ assert missing == ["__ming_image__"]
+ assert unexpected == []
+ assert model.state_dict()["__ming_image__"].device.type == "cpu"
+ saved_sd = model_config.process_unet_state_dict_for_saving(model.state_dict())
+ reloaded = model_config_from_unet(saved_sd, "model.diffusion_model.")
+
+ assert type(reloaded) is comfy.supported_models.MingImage
+ assert reloaded.latent_format.scale_factor == model_config.latent_format.scale_factor
+ assert reloaded.sampling_settings == model_config.sampling_settings
+ assert reloaded.unet_config == model_config.unet_config
+
+ unprefixed = {k.removeprefix("model.diffusion_model."): v for k, v in saved_sd.items()}
+ assert type(model_config_from_unet(unprefixed, "")) is comfy.supported_models.MingImage
+ model.load_state_dict(reloaded.process_unet_state_dict(unprefixed), strict=True, assign=True)
+
def test_unet_config_and_required_keys_combination_is_unique(self):
"""Each model in the registry must have a unique combination of
``unet_config`` and ``required_keys``. If two models share the same