The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Inpainting for z image fun control. Use the ZImageFunControlnet node. (#11346)
comfyanonymous645ee1881e73
diff --git a/comfy_extras/nodes_model_patch.py b/comfy_extras/nodes_model_patch.pyindex fdd5d0d3f..2a0cfcf18 100644--- a/comfy_extras/nodes_model_patch.py+++ b/comfy_extras/nodes_model_patch.py@@ -313,22 +313,46 @@ class ZImageControlPatch:self.inpaint_image = inpaint_imageself.mask = maskself.strength = strength- self.encoded_image = self.encode_latent_cond(image)- self.encoded_image_size = (image.shape[1], image.shape[2])- self.temp_data = None+ self.is_inpaint = self.model_patch.model.additional_in_dim > 0- def encode_latent_cond(self, control_image, inpaint_image=None):- latent_image = comfy.latent_formats.Flux().process_in(self.vae.encode(control_image))- if self.model_patch.model.additional_in_dim > 0:- if self.mask is None:- mask_ = torch.zeros_like(latent_image)[:, :1]+ skip_encoding = False+ if self.image is not None and self.inpaint_image is not None:+ if self.image.shape != self.inpaint_image.shape:+ skip_encoding = True++ if skip_encoding:+ self.encoded_image = None+ else:+ self.encoded_image = self.encode_latent_cond(self.image, self.inpaint_image)+ if self.image is None:+ self.encoded_image_size = (self.inpaint_image.shape[1], self.inpaint_image.shape[2])else:- mask_ = comfy.utils.common_upscale(self.mask.mean(dim=1, keepdim=True), latent_image.shape[-1], latent_image.shape[-2], "bilinear", "none")+ self.encoded_image_size = (self.image.shape[1], self.image.shape[2])+ self.temp_data = None++ def encode_latent_cond(self, control_image=None, inpaint_image=None):+ latent_image = None+ if control_image is not None:+ latent_image = comfy.latent_formats.Flux().process_in(self.vae.encode(control_image))++ if self.is_inpaint:if inpaint_image is None:inpaint_image = torch.ones_like(control_image) * 0.5+ if self.mask is not None:+ mask_inpaint = comfy.utils.common_upscale(self.mask.view(self.mask.shape[0], -1, self.mask.shape[-2], self.mask.shape[-1]).mean(dim=1, keepdim=True), inpaint_image.shape[-2], inpaint_image.shape[-3], "bilinear", "center")+ inpaint_image = ((inpaint_image - 0.5) * mask_inpaint.movedim(1, -1).round()) + 0.5+inpaint_image_latent = comfy.latent_formats.Flux().process_in(self.vae.encode(inpaint_image))+ if self.mask is None:+ mask_ = torch.zeros_like(inpaint_image_latent)[:, :1]+ else:+ mask_ = comfy.utils.common_upscale(self.mask.view(self.mask.shape[0], -1, self.mask.shape[-2], self.mask.shape[-1]).mean(dim=1, keepdim=True), inpaint_image_latent.shape[-1], inpaint_image_latent.shape[-2], "nearest", "center")++ if latent_image is None:+ latent_image = comfy.latent_formats.Flux().process_in(self.vae.encode(torch.ones_like(inpaint_image) * 0.5))+return torch.cat([latent_image, mask_, inpaint_image_latent], dim=1)else:return latent_image@@ -344,13 +368,18 @@ class ZImageControlPatch:block_type = kwargs.get("block_type", "")spacial_compression = self.vae.spacial_compression_encode()if self.encoded_image is None or self.encoded_image_size != (x.shape[-2] * spacial_compression, x.shape[-1] * spacial_compression):- image_scaled = comfy.utils.common_upscale(self.image.movedim(-1, 1), x.shape[-1] * spacial_compression, x.shape[-2] * spacial_compression, "area", "center")+ image_scaled = None+ if self.image is not None:+ image_scaled = comfy.utils.common_upscale(self.image.movedim(-1, 1), x.shape[-1] * spacial_compression, x.shape[-2] * spacial_compression, "area", "center").movedim(1, -1)+ self.encoded_image_size = (image_scaled.shape[-3], image_scaled.shape[-2])+inpaint_scaled = Noneif self.inpaint_image is not None:inpaint_scaled = comfy.utils.common_upscale(self.inpaint_image.movedim(-1, 1), x.shape[-1] * spacial_compression, x.shape[-2] * spacial_compression, "area", "center").movedim(1, -1)+ self.encoded_image_size = (inpaint_scaled.shape[-3], inpaint_scaled.shape[-2])+loaded_models = comfy.model_management.loaded_models(only_currently_used=True)- self.encoded_image = self.encode_latent_cond(image_scaled.movedim(1, -1), inpaint_scaled)- self.encoded_image_size = (image_scaled.shape[-2], image_scaled.shape[-1])+ self.encoded_image = self.encode_latent_cond(image_scaled, inpaint_scaled)comfy.model_management.load_models_gpu(loaded_models)cnet_blocks = self.model_patch.model.n_control_layers@@ -391,7 +420,8 @@ class ZImageControlPatch:def to(self, device_or_dtype):if isinstance(device_or_dtype, torch.device):- self.encoded_image = self.encoded_image.to(device_or_dtype)+ if self.encoded_image is not None:+ self.encoded_image = self.encoded_image.to(device_or_dtype)self.temp_data = Nonereturn self@@ -414,9 +444,12 @@ class QwenImageDiffsynthControlnet:CATEGORY = "advanced/loaders/qwen"- def diffsynth_controlnet(self, model, model_patch, vae, image, strength, mask=None):+ def diffsynth_controlnet(self, model, model_patch, vae, image=None, strength=1.0, inpaint_image=None, mask=None):model_patched = model.clone()- image = image[:, :, :, :3]+ if image is not None:+ image = image[:, :, :, :3]+ if inpaint_image is not None:+ inpaint_image = inpaint_image[:, :, :, :3]if mask is not None:if mask.ndim == 3:mask = mask.unsqueeze(1)@@ -425,13 +458,24 @@ class QwenImageDiffsynthControlnet:mask = 1.0 - maskif isinstance(model_patch.model, comfy.ldm.lumina.controlnet.ZImage_Control):- patch = ZImageControlPatch(model_patch, vae, image, strength, mask=mask)+ patch = ZImageControlPatch(model_patch, vae, image, strength, inpaint_image=inpaint_image, mask=mask)model_patched.set_model_noise_refiner_patch(patch)model_patched.set_model_double_block_patch(patch)else:model_patched.set_model_double_block_patch(DiffSynthCnetPatch(model_patch, vae, image, strength, mask))return (model_patched,)+class ZImageFunControlnet(QwenImageDiffsynthControlnet):+ @classmethod+ def INPUT_TYPES(s):+ return {"required": { "model": ("MODEL",),+ "model_patch": ("MODEL_PATCH",),+ "vae": ("VAE",),+ "strength": ("FLOAT", {"default": 1.0, "min": -10.0, "max": 10.0, "step": 0.01}),+ },+ "optional": {"image": ("IMAGE",), "inpaint_image": ("IMAGE",), "mask": ("MASK",)}}++ CATEGORY = "advanced/loaders/zimage"class UsoStyleProjectorPatch:def __init__(self, model_patch, encoded_image):@@ -479,5 +523,6 @@ class USOStyleReference:NODE_CLASS_MAPPINGS = {"ModelPatchLoader": ModelPatchLoader,"QwenImageDiffsynthControlnet": QwenImageDiffsynthControlnet,+ "ZImageFunControlnet": ZImageFunControlnet,"USOStyleReference": USOStyleReference,}