The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Support Qwen-Image 2.1 union fun controlnet (#16519)
Jukka Seppänen3b3939a78878
diff --git a/comfy/controlnet.py b/comfy/controlnet.pyindex a7c96ac05..a157a9dff 100644--- a/comfy/controlnet.py+++ b/comfy/controlnet.py@@ -800,7 +800,7 @@ def load_controlnet_state_dict(state_dict, model=None, model_options={}):return load_controlnet_qwen_instantx(controlnet_data, model_options=model_options)elif "controlnet_x_embedder.weight" in controlnet_data:return load_controlnet_flux_instantx(controlnet_data, model_options=model_options)- elif "control_blocks.0.after_proj.weight" in controlnet_data and "control_img_in.weight" in controlnet_data:+ elif "control_blocks.0.after_proj.weight" in controlnet_data and "control_blocks.0.img_mod.1.weight" in controlnet_data and "control_img_in.weight" in controlnet_data:return load_controlnet_qwen_fun(controlnet_data, model_options=model_options)elif "controlnet_blocks.0.linear.weight" in controlnet_data: #mistoline fluxdiff --git a/comfy/ldm/qwen_image21/model.py b/comfy/ldm/qwen_image21/model.pyindex f38874ea2..8e2d6ad5a 100644--- a/comfy/ldm/qwen_image21/model.py+++ b/comfy/ldm/qwen_image21/model.py@@ -163,6 +163,36 @@ class LastLayer(nn.Module):return comfy.quant_ops.ck.adaln(x, scale, torch.zeros_like(scale[:1]), self.norm.eps)+class QwenImage21FunControlBlock(QwenImage21TransformerBlock):+ def __init__(self, dim, num_attention_heads, attention_head_dim, mlp_ratio=3, eps=1e-6, fused_mlp=True, first=False, dtype=None, device=None, operations=None):+ super().__init__(dim, num_attention_heads, attention_head_dim, mlp_ratio, eps, fused_mlp, dtype=dtype, device=device, operations=operations)+ if first:+ self.before_proj = operations.Linear(dim, dim, dtype=dtype, device=device)+ self.after_proj = operations.Linear(dim, dim, dtype=dtype, device=device)+++class QwenImage21FunControl(nn.Module):+ # VACE-style branch: a stream of block copies started from the embedded joint sequence, each skip added after one base block+ def __init__(self, num_blocks=16, control_in_dim=129, inner_dim=4096, attention_head_dim=128, mlp_ratio=3, eps=1e-6, fused_mlp=True, dtype=None, device=None, operations=None):+ super().__init__()+ self.control_img_in = operations.Linear(control_in_dim, inner_dim, dtype=dtype, device=device)+ self.control_blocks = nn.ModuleList([+ QwenImage21FunControlBlock(inner_dim, inner_dim // attention_head_dim, attention_head_dim, mlp_ratio, eps, fused_mlp, first=i == 0, dtype=dtype, device=device, operations=operations)+ for i in range(num_blocks)+ ])++ def init_stream(self, x, control, prefix_len):+ # control latents fill the target rows, text and reference rows stay zero+ joint = torch.zeros_like(x)+ joint[:, prefix_len:] = self.control_img_in(control)+ return self.control_blocks[0].before_proj(joint) + x++ def step(self, index, c, mod, pe, attn_fn, prefix_len, transformer_options={}):+ block = self.control_blocks[index]+ c = block(c, mod, pe, attn_fn, prefix_len, transformer_options)+ return c, block.after_proj(c)++def block_causal_attention(segments, transformer_options={}, cache=None, block_index=0, prefix_len=0):# segments: (start, end, mask); text segments get a causal mask, image blocks attend to everything before their enddef attn(q, k, v, heads, preferred_attention=None):@@ -359,7 +389,8 @@ class QwenImage21Transformer2DModel(nn.Module):if ("single_block", i) in blocks_replace:def block_wrap(args):return {"img": block(args["img"], mod, args["pe"], attn_fn, prefix_len, args["transformer_options"])}- hidden_states = blocks_replace[("single_block", i)]({"img": hidden_states, "vec": temb, "pe": pe, "transformer_options": transformer_options}, {"original_block": block_wrap})["img"]+ args = {"img": hidden_states, "vec": temb, "pe": pe, "mod": mod, "attn_fn": attn_fn, "prefix_len": prefix_len, "transformer_options": transformer_options}+ hidden_states = blocks_replace[("single_block", i)](args, {"original_block": block_wrap})["img"]else:hidden_states = block(hidden_states, mod, pe, attn_fn, prefix_len, transformer_options)for p in patches.get("single_block", []):diff --git a/comfy_extras/nodes_model_patch.py b/comfy_extras/nodes_model_patch.pyindex d9836ee32..30374ce1e 100644--- a/comfy_extras/nodes_model_patch.py+++ b/comfy_extras/nodes_model_patch.py@@ -6,6 +6,8 @@ import folder_pathsimport comfy.utilsimport comfy.opsimport comfy.model_management+import comfy.model_prefetch+import comfy.patcher_extensionimport comfy.storageimport comfy.ldm.common_ditimport comfy.latent_formats@@ -13,6 +15,7 @@ import comfy.ldm.lumina.controlnetimport comfy.ldm.supir.supir_modulesimport comfy.ldm.anima.llliteimport comfy.ldm.minimax.controlnet+import comfy.ldm.qwen_image21.modelimport comfy.ldm.wan.uni3cimport comfy.ldm.lightricks.duration_headfrom comfy.ldm.wan.model_multitalk import WanMultiTalkAttentionBlock, MultiTalkAudioProjModel@@ -230,6 +233,17 @@ def z_image_convert(sd):return out_sd+def dit_patch_operations(sd):+ # quantized files keep their layers quantized with bf16 compute, others follow the usual bf16/fp32 dtype pick+ quant = comfy.utils.detect_layer_quantization(sd, "")+ if quant is not None:+ return torch.bfloat16, comfy.ops.mixed_precision_ops(quant, torch.bfloat16)+ load_device = comfy.model_management.get_torch_device()+ dtype = comfy.model_management.unet_dtype(model_params=-1, supported_dtypes=[torch.bfloat16, torch.float32], weight_dtype=comfy.utils.weight_dtype(sd))+ manual_cast_dtype = comfy.model_management.unet_manual_cast(dtype, load_device, supported_dtypes=[torch.bfloat16, torch.float32])+ return dtype, comfy.ops.pick_operations(dtype, manual_cast_dtype, load_device=load_device)++class ModelPatchLoader:@classmethoddef INPUT_TYPES(s):@@ -270,22 +284,27 @@ class ModelPatchLoader:if torch.count_nonzero(ref_weight) == 0:config['broken'] = Truemodel = comfy.ldm.lumina.controlnet.ZImage_Control(device=comfy.model_management.unet_offload_device(), dtype=dtype, operations=comfy.ops.manual_cast, **config)+ elif 'control_img_in.weight' in sd and 'control_blocks.0.img_mlp.out.weight' in sd: # Qwen Image 2.1 Fun ControlNet+ dtype, operations = dit_patch_operations(sd)+ inner_dim = sd["control_img_in.weight"].shape[0]+ gate_up = sd.get("control_blocks.0.img_mlp.gate_up.weight", None)+ hidden_dim = gate_up.shape[0] // 2 if gate_up is not None else sd["control_blocks.0.img_mlp.proj.weight"].shape[0]+ num_blocks = 0+ while "control_blocks.{}.after_proj.weight".format(num_blocks) in sd:+ num_blocks += 1+ model = comfy.ldm.qwen_image21.model.QwenImage21FunControl(+ num_blocks=num_blocks,+ control_in_dim=129,+ inner_dim=inner_dim,+ attention_head_dim=sd["control_blocks.0.attn.norm_q.weight"].shape[0],+ mlp_ratio=hidden_dim // inner_dim,+ fused_mlp=gate_up is not None,+ operations=operations,+ device=comfy.model_management.unet_offload_device(),+ dtype=dtype,+ )elif comfy.ldm.minimax.controlnet.is_minimax_h3_fun_state_dict(sd):- load_device = comfy.model_management.get_torch_device()- quant = comfy.utils.detect_layer_quantization(sd, "")- if quant is not None:- dtype = torch.bfloat16- operations = comfy.ops.mixed_precision_ops(quant, dtype)- else:- dtype = comfy.model_management.unet_dtype(- model_params=-1,- supported_dtypes=[torch.bfloat16, torch.float32],- weight_dtype=comfy.utils.weight_dtype(sd),- )- manual_cast_dtype = comfy.model_management.unet_manual_cast(- dtype, load_device, supported_dtypes=[torch.bfloat16, torch.float32])- operations = comfy.ops.pick_operations(dtype, manual_cast_dtype, load_device=load_device)-+ dtype, operations = dit_patch_operations(sd)num_blocks = 0while "control_blocks.{}.after_proj.weight".format(num_blocks) in sd:num_blocks += 1@@ -428,13 +447,19 @@ class AnimaLLLiteApply:return (model_patched,)+def in_sigma_range(transformer_options, sigma_range):+ sigma = float(transformer_options["sigmas"].flatten()[0])+ return sigma_range[1] <= sigma <= sigma_range[0]++class DiffSynthCnetPatch:- def __init__(self, model_patch, vae, image, strength, mask=None):+ def __init__(self, model_patch, vae, image, strength, mask=None, sigma_range=(float("inf"), 0.0)):self.model_patch = model_patchself.vae = vaeself.image = imageself.strength = strengthself.mask = mask+ self.sigma_range = sigma_rangeself.encoded_image = model_patch.model.process_input_latent_image(self.encode_latent_cond(image))self.encoded_image_size = (image.shape[1], image.shape[2])@@ -451,6 +476,8 @@ class DiffSynthCnetPatch:return latent_imagedef __call__(self, kwargs):+ if not in_sigma_range(kwargs["transformer_options"], self.sigma_range):+ return kwargsx = kwargs.get("x")img = kwargs.get("img")block_index = kwargs.get("block_index")@@ -475,8 +502,9 @@ class DiffSynthCnetPatch:return [self.model_patch]class ZImageControlPatch:- def __init__(self, model_patch, vae, image, strength, inpaint_image=None, mask=None):+ def __init__(self, model_patch, vae, image, strength, inpaint_image=None, mask=None, sigma_range=(float("inf"), 0.0)):self.model_patch = model_patch+ self.sigma_range = sigma_rangeself.vae = vaeself.image = imageself.inpaint_image = inpaint_image@@ -527,6 +555,8 @@ class ZImageControlPatch:return latent_imagedef __call__(self, kwargs):+ if not in_sigma_range(kwargs["transformer_options"], self.sigma_range):+ return kwargsx = kwargs.get("x")img = kwargs.get("img")img_input = kwargs.get("img_input")@@ -597,6 +627,141 @@ class ZImageControlPatch:def models(self):return [self.model_patch]+class QwenImage21FunControlPatch:+ def __init__(self, model_patch, vae, image, strength, inpaint_image=None, mask=None, sigma_range=(float("inf"), 0.0)):+ self.model_patch = model_patch+ self.vae = vae+ self.image = image+ self.inpaint_image = inpaint_image+ self.mask = mask+ self.strength = strength+ self.sigma_range = sigma_range+ self.active = False+ self.injection_layers = None+ self.control = None+ self.stream = None+ self.pristine = None++ def prepare(self, h, w):+ # 129 channels per target token: control latents | keep mask | masked-image latents, zeros where not given+ if self.control is not None and self.control.shape[-2:] == (h, w):+ return+ width, height = w * self.vae.spacial_compression_encode(), h * self.vae.spacial_compression_encode()+ latent_format = comfy.latent_formats.QwenImage21()+ loaded_models = comfy.model_management.loaded_models(only_currently_used=True)+ try:+ control = torch.zeros(1, latent_format.latent_channels, h, w)+ if self.image is not None:+ image = comfy.utils.common_upscale(self.image[:1].movedim(-1, 1), width, height, "bicubic", "disabled")+ control = latent_format.process_in(self.vae.encode(image.movedim(1, -1))).float().cpu()+ regen = torch.ones(1, 1, height, width)+ if self.mask is not None:+ mask = self.mask.reshape(-1, 1, *self.mask.shape[-2:])[:1].float().cpu()+ regen = (comfy.utils.common_upscale(mask, width, height, "bilinear", "disabled") >= 0.5).float()+ inpaint = torch.zeros_like(control)+ if self.inpaint_image is not None:+ # regenerated pixels at mid-gray, the zero of the VAE's [-1, 1] input+ image = comfy.utils.common_upscale(self.inpaint_image[:1].movedim(-1, 1), width, height, "bicubic", "disabled").float().cpu()+ inpaint = latent_format.process_in(self.vae.encode((image * (1 - regen) + 0.5 * regen).movedim(1, -1))).float().cpu()+ keep = 1 - torch.nn.functional.interpolate(regen, size=(h, w), mode="nearest")+ finally:+ comfy.model_management.load_models_gpu(loaded_models)+ self.control = torch.cat([control, keep, inpaint], dim=1)++ def diffusion_model_wrapper(self, executor, x, timestep, context, ref_latents, image_slots, transformer_options, **kwargs):+ self.active = in_sigma_range(transformer_options, self.sigma_range)+ if self.active:+ with comfy.model_prefetch.pause_malloc_graph():+ self.prepare(*x.shape[-2:])+ else:+ # outside the range the block patches are dropped so the model can use its prefix cache+ dit = transformer_options.get("patches_replace", {}).get("dit", {})+ dit = {k: p.previous if isinstance(p, QwenImage21FunControlBlockPatch) and p.control_patch is self else p for k, p in dit.items()}+ dit = {k: p for k, p in dit.items() if p is not None}+ transformer_options = {**transformer_options, "patches_replace": {**transformer_options["patches_replace"], "dit": dit}}+ try:+ return executor(x, timestep, context, ref_latents, image_slots, transformer_options, **kwargs)+ finally:+ self.stream = None+ self.pristine = None++ def before_block(self, block_index, args):+ if self.active and block_index == self.injection_layers[0]:+ # the base block updates its input in place+ self.pristine = args["img"].clone()++ def after_block(self, block_index, args, out):+ if not self.active:+ return out+ model = self.model_patch.model+ index = self.injection_layers.index(block_index)+ if index == 0:+ self.control = self.control.to(out["img"].device, out["img"].dtype)+ self.stream = model.init_stream(self.pristine, self.control.flatten(2).transpose(1, 2), args["prefix_len"])+ self.pristine = None+ self.stream, skip = model.step(index, self.stream, args["mod"], args["pe"], args["attn_fn"], args["prefix_len"], args["transformer_options"])+ out["img"].add_(skip, alpha=self.strength)+ return out++ def to(self, device_or_dtype):+ if isinstance(device_or_dtype, torch.device):+ if self.control is not None:+ self.control = self.control.to(device_or_dtype)+ self.stream = None+ return self++ def cleanup(self):+ self.control = None+ self.stream = None+ self.pristine = None+ self.active = False++ def models(self):+ return [self.model_patch]++ def register(self, model):+ # the control blocks pair with every (num_base / num_control)-th base block+ num_base = len(model.get_model_object("diffusion_model").transformer_blocks)+ self.injection_layers = list(range(0, num_base, num_base // len(self.model_patch.model.control_blocks)))+ model.add_wrapper(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, self.diffusion_model_wrapper)+ blocks_replace = model.model_options.get("transformer_options", {}).get("patches_replace", {}).get("dit", {})+ for block_index in self.injection_layers:+ previous = blocks_replace.get(("single_block", block_index))+ model.set_model_patch_replace(QwenImage21FunControlBlockPatch(self, block_index, previous), "dit", "single_block", block_index)+++class QwenImage21FunControlBlockPatch:+ def __init__(self, control_patch, block_index, previous):+ self.control_patch = control_patch+ self.block_index = block_index+ self.previous = previous++ def __call__(self, args, extra_args):+ # control state stays outside the base block's allocation scope+ with comfy.model_prefetch.pause_malloc_graph():+ self.control_patch.before_block(self.block_index, args)+ out = extra_args["original_block"](args) if self.previous is None else self.previous(args, extra_args)+ with comfy.model_prefetch.pause_malloc_graph():+ return self.control_patch.after_block(self.block_index, args, out)++ def to(self, device_or_dtype):+ self.control_patch.to(device_or_dtype)+ if hasattr(self.previous, "to"):+ self.previous = self.previous.to(device_or_dtype)+ return self++ def cleanup(self):+ self.control_patch.cleanup()+ if hasattr(self.previous, "cleanup"):+ self.previous.cleanup()++ def models(self):+ models = self.control_patch.models()+ if hasattr(self.previous, "models"):+ models += self.previous.models()+ return models++class QwenImageDiffsynthControlnet:@classmethoddef INPUT_TYPES(s):@@ -606,19 +771,28 @@ class QwenImageDiffsynthControlnet:"image": ("IMAGE",),"strength": ("FLOAT", {"default": 1.0, "min": -10.0, "max": 10.0, "step": 0.01}),},- "optional": {"mask": ("MASK",)}}+ "optional": {"mask": ("MASK",),+ "start_percent": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001, "advanced": True}),+ "end_percent": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001, "advanced": True})}}RETURN_TYPES = ("MODEL",)FUNCTION = "diffsynth_controlnet"EXPERIMENTAL = TrueCATEGORY = "model/patch/qwen"- def diffsynth_controlnet(self, model, model_patch, vae, image=None, strength=1.0, inpaint_image=None, mask=None):+ def diffsynth_controlnet(self, model, model_patch, vae, image=None, strength=1.0, inpaint_image=None, mask=None, start_percent=0.0, end_percent=1.0):+ if strength == 0 or (image is None and inpaint_image is None and mask is None):+ return (model,)model_patched = model.clone()+ model_sampling = model.get_model_object("model_sampling")+ sigma_range = (float(model_sampling.percent_to_sigma(start_percent)), float(model_sampling.percent_to_sigma(end_percent)))if image is not None:image = image[:, :, :, :3]if inpaint_image is not None:inpaint_image = inpaint_image[:, :, :, :3]+ if isinstance(model_patch.model, comfy.ldm.qwen_image21.model.QwenImage21FunControl):+ QwenImage21FunControlPatch(model_patch, vae, image, strength, inpaint_image=inpaint_image, mask=mask, sigma_range=sigma_range).register(model_patched)+ return (model_patched,)if mask is not None:if mask.ndim == 3:mask = mask.unsqueeze(1)@@ -627,11 +801,11 @@ class QwenImageDiffsynthControlnet:mask = 1.0 - maskif isinstance(model_patch.model, comfy.ldm.lumina.controlnet.ZImage_Control):- patch = ZImageControlPatch(model_patch, vae, image, strength, inpaint_image=inpaint_image, mask=mask)+ patch = ZImageControlPatch(model_patch, vae, image, strength, inpaint_image=inpaint_image, mask=mask, sigma_range=sigma_range)model_patched.set_model_noise_refiner_patch(patch)model_patched.set_model_double_block_patch(patch)else:- model_patched.set_model_double_block_patch(DiffSynthCnetPatch(model_patch, vae, image, strength, mask))+ model_patched.set_model_double_block_patch(DiffSynthCnetPatch(model_patch, vae, image, strength, mask, sigma_range=sigma_range))return (model_patched,)class ZImageFunControlnet(QwenImageDiffsynthControlnet):@@ -642,9 +816,13 @@ class ZImageFunControlnet(QwenImageDiffsynthControlnet):"vae": ("VAE",),"strength": ("FLOAT", {"default": 1.0, "min": -10.0, "max": 10.0, "step": 0.01}),},- "optional": {"image": ("IMAGE",), "inpaint_image": ("IMAGE",), "mask": ("MASK",)}}+ "optional": {"image": ("IMAGE",), "inpaint_image": ("IMAGE",), "mask": ("MASK",),+ "start_percent": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001, "advanced": True}),+ "end_percent": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001, "advanced": True})}}- CATEGORY = "model/patch/z-image"+ CATEGORY = "model/patch"+ SEARCH_ALIASES = ["z-image controlnet", "qwen image 2.1 controlnet", "controlnet union", "fun controlnet"]+ DESCRIPTION = "Applies a Z-Image or Qwen Image 2.1 Fun ControlNet loaded with Load Model Patch."class WanUni3CCnetPatch:def __init__(self, model_patch, render_video, vae, latent_format, strength, sigma_start, sigma_end):@@ -957,7 +1135,7 @@ NODE_CLASS_MAPPINGS = {NODE_DISPLAY_NAME_MAPPINGS = {"ModelPatchLoader": "Load Model Patch","QwenImageDiffsynthControlnet": "Apply Qwen Image DiffSynth ControlNet",- "ZImageFunControlnet": "Apply Z-Image Fun ControlNet",+ "ZImageFunControlnet": "Apply Fun ControlNet","WanUni3CControlnetApply": "Apply Wan Uni3C ControlNet","USOStyleReference": "Apply USO Style Reference","SUPIRApply": "Apply SUPIR Patch",