The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Fix saving/loading ming image model breaking detection. (#16514)
comfyanonymous41c83ee79f46
diff --git a/comfy/ldm/lumina/model.py b/comfy/ldm/lumina/model.pyindex 62ccdad11..bc9528875 100644--- a/comfy/ldm/lumina/model.py+++ b/comfy/ldm/lumina/model.py@@ -479,6 +479,9 @@ class NextDiT(nn.Module):self.pad_tokens_multiple = pad_tokens_multipleself.masked_pad_multiple = masked_pad_multiple+ if image_model == "ming_image":+ self.register_buffer("__ming_image__", torch.empty(0))+self.x_embedder = operation_settings.get("operations").Linear(in_features=patch_size * patch_size * in_channels,out_features=dim,diff --git a/comfy/model_detection.py b/comfy/model_detection.pyindex 6a1d288ec..46332edff 100644--- a/comfy/model_detection.py+++ b/comfy/model_detection.py@@ -612,9 +612,9 @@ def detect_unet_config(state_dict, key_prefix, metadata=None):dec_cond_key = '{}dec_net.cond_embed.weight'.format(key_prefix)if '{}cap_pad_token'.format(key_prefix) in state_dict_keys:dit_config["pad_tokens_multiple"] = 32- # Ming-Image: Z-Image keys. Design ships without the learned pad tokens; Layer has them and only its repack metadata tells it apart.+ # Ming-Image Design has no learned pad tokens; Layer needs a saved marker or repack metadata.ming_metadata = metadata is not None and "config" in metadata and json.loads(metadata["config"]).get("transformer", {}).get("image_model") == "ming_image"- if ming_metadata or ("pad_tokens_multiple" not in dit_config and dec_cond_key not in state_dict_keys):+ if '{}__ming_image__'.format(key_prefix) in state_dict_keys or ming_metadata or ("pad_tokens_multiple" not in dit_config and dec_cond_key not in state_dict_keys):dit_config["image_model"] = "ming_image"if "pad_tokens_multiple" not in dit_config:dit_config["masked_pad_multiple"] = 32diff --git a/tests-unit/comfy_test/model_detection_test.py b/tests-unit/comfy_test/model_detection_test.pyindex 94da2fdf6..de4fe8e09 100644--- a/tests-unit/comfy_test/model_detection_test.py+++ b/tests-unit/comfy_test/model_detection_test.py@@ -3,6 +3,8 @@ from collections import defaultdictimport torchfrom comfy.model_detection import detect_unet_config, model_config_from_unet, model_config_from_unet_config+from comfy.ldm.lumina.model import NextDiT+import comfy.opsimport comfy.supported_models@@ -301,6 +303,37 @@ class TestModelDetection:assert unet_config["proj_in_channels_shape"] == 2048assert unet_config["proj_in_channels_structure"] == 1024+ def test_ming_image_save_preserves_identity_without_metadata(self):+ for learned_padding in (False, True):+ sd = {+ "cap_embedder.1.weight": torch.empty(3840, 2560, device="meta"),+ "noise_refiner.0.attention.k_norm.weight": torch.empty(128, device="meta"),+ }+ if learned_padding:+ sd["cap_pad_token"] = torch.empty(1, 3840, device="meta")+ sd["x_pad_token"] = torch.empty(1, 3840, device="meta")+ assert type(model_config_from_unet(sd, "")) is comfy.supported_models.ZImage++ model_config = model_config_from_unet(sd, "", metadata={"config": '{"transformer": {"image_model": "ming_image"}}'})+ model = NextDiT(**model_config.unet_config, device="meta", operations=comfy.ops.manual_cast)+ original_sd = model.state_dict()+ del original_sd["__ming_image__"]+ missing, unexpected = model.load_state_dict(original_sd, strict=False, assign=True)+ assert missing == ["__ming_image__"]+ assert unexpected == []+ assert model.state_dict()["__ming_image__"].device.type == "cpu"+ saved_sd = model_config.process_unet_state_dict_for_saving(model.state_dict())+ reloaded = model_config_from_unet(saved_sd, "model.diffusion_model.")++ assert type(reloaded) is comfy.supported_models.MingImage+ assert reloaded.latent_format.scale_factor == model_config.latent_format.scale_factor+ assert reloaded.sampling_settings == model_config.sampling_settings+ assert reloaded.unet_config == model_config.unet_config++ unprefixed = {k.removeprefix("model.diffusion_model."): v for k, v in saved_sd.items()}+ assert type(model_config_from_unet(unprefixed, "")) is comfy.supported_models.MingImage+ model.load_state_dict(reloaded.process_unet_state_dict(unprefixed), strict=True, assign=True)+def test_unet_config_and_required_keys_combination_is_unique(self):"""Each model in the registry must have a unique combination of``unet_config`` and ``required_keys``. If two models share the same