Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Support Qwen-Image 2.1 union fun controlnet (#16519)

Jukka Seppänen3b3939a78878
diff --git a/comfy/controlnet.py b/comfy/controlnet.py
index a7c96ac05..a157a9dff 100644
--- a/comfy/controlnet.py
+++ b/comfy/controlnet.py
@@ -800,7 +800,7 @@ def load_controlnet_state_dict(state_dict, model=None, model_options={}):
return load_controlnet_qwen_instantx(controlnet_data, model_options=model_options)
elif "controlnet_x_embedder.weight" in controlnet_data:
return load_controlnet_flux_instantx(controlnet_data, model_options=model_options)
- elif "control_blocks.0.after_proj.weight" in controlnet_data and "control_img_in.weight" in controlnet_data:
+ elif "control_blocks.0.after_proj.weight" in controlnet_data and "control_blocks.0.img_mod.1.weight" in controlnet_data and "control_img_in.weight" in controlnet_data:
return load_controlnet_qwen_fun(controlnet_data, model_options=model_options)
elif "controlnet_blocks.0.linear.weight" in controlnet_data: #mistoline flux
diff --git a/comfy/ldm/qwen_image21/model.py b/comfy/ldm/qwen_image21/model.py
index f38874ea2..8e2d6ad5a 100644
--- a/comfy/ldm/qwen_image21/model.py
+++ b/comfy/ldm/qwen_image21/model.py
@@ -163,6 +163,36 @@ class LastLayer(nn.Module):
return comfy.quant_ops.ck.adaln(x, scale, torch.zeros_like(scale[:1]), self.norm.eps)
+class QwenImage21FunControlBlock(QwenImage21TransformerBlock):
+ def __init__(self, dim, num_attention_heads, attention_head_dim, mlp_ratio=3, eps=1e-6, fused_mlp=True, first=False, dtype=None, device=None, operations=None):
+ super().__init__(dim, num_attention_heads, attention_head_dim, mlp_ratio, eps, fused_mlp, dtype=dtype, device=device, operations=operations)
+ if first:
+ self.before_proj = operations.Linear(dim, dim, dtype=dtype, device=device)
+ self.after_proj = operations.Linear(dim, dim, dtype=dtype, device=device)
+
+
+class QwenImage21FunControl(nn.Module):
+ # VACE-style branch: a stream of block copies started from the embedded joint sequence, each skip added after one base block
+ def __init__(self, num_blocks=16, control_in_dim=129, inner_dim=4096, attention_head_dim=128, mlp_ratio=3, eps=1e-6, fused_mlp=True, dtype=None, device=None, operations=None):
+ super().__init__()
+ self.control_img_in = operations.Linear(control_in_dim, inner_dim, dtype=dtype, device=device)
+ self.control_blocks = nn.ModuleList([
+ QwenImage21FunControlBlock(inner_dim, inner_dim // attention_head_dim, attention_head_dim, mlp_ratio, eps, fused_mlp, first=i == 0, dtype=dtype, device=device, operations=operations)
+ for i in range(num_blocks)
+ ])
+
+ def init_stream(self, x, control, prefix_len):
+ # control latents fill the target rows, text and reference rows stay zero
+ joint = torch.zeros_like(x)
+ joint[:, prefix_len:] = self.control_img_in(control)
+ return self.control_blocks[0].before_proj(joint) + x
+
+ def step(self, index, c, mod, pe, attn_fn, prefix_len, transformer_options={}):
+ block = self.control_blocks[index]
+ c = block(c, mod, pe, attn_fn, prefix_len, transformer_options)
+ return c, block.after_proj(c)
+
+
def block_causal_attention(segments, transformer_options={}, cache=None, block_index=0, prefix_len=0):
# segments: (start, end, mask); text segments get a causal mask, image blocks attend to everything before their end
def attn(q, k, v, heads, preferred_attention=None):
@@ -359,7 +389,8 @@ class QwenImage21Transformer2DModel(nn.Module):
if ("single_block", i) in blocks_replace:
def block_wrap(args):
return {"img": block(args["img"], mod, args["pe"], attn_fn, prefix_len, args["transformer_options"])}
- hidden_states = blocks_replace[("single_block", i)]({"img": hidden_states, "vec": temb, "pe": pe, "transformer_options": transformer_options}, {"original_block": block_wrap})["img"]
+ args = {"img": hidden_states, "vec": temb, "pe": pe, "mod": mod, "attn_fn": attn_fn, "prefix_len": prefix_len, "transformer_options": transformer_options}
+ hidden_states = blocks_replace[("single_block", i)](args, {"original_block": block_wrap})["img"]
else:
hidden_states = block(hidden_states, mod, pe, attn_fn, prefix_len, transformer_options)
for p in patches.get("single_block", []):
diff --git a/comfy_extras/nodes_model_patch.py b/comfy_extras/nodes_model_patch.py
index d9836ee32..30374ce1e 100644
--- a/comfy_extras/nodes_model_patch.py
+++ b/comfy_extras/nodes_model_patch.py
@@ -6,6 +6,8 @@ import folder_paths
import comfy.utils
import comfy.ops
import comfy.model_management
+import comfy.model_prefetch
+import comfy.patcher_extension
import comfy.storage
import comfy.ldm.common_dit
import comfy.latent_formats
@@ -13,6 +15,7 @@ import comfy.ldm.lumina.controlnet
import comfy.ldm.supir.supir_modules
import comfy.ldm.anima.lllite
import comfy.ldm.minimax.controlnet
+import comfy.ldm.qwen_image21.model
import comfy.ldm.wan.uni3c
import comfy.ldm.lightricks.duration_head
from comfy.ldm.wan.model_multitalk import WanMultiTalkAttentionBlock, MultiTalkAudioProjModel
@@ -230,6 +233,17 @@ def z_image_convert(sd):
return out_sd
+def dit_patch_operations(sd):
+ # quantized files keep their layers quantized with bf16 compute, others follow the usual bf16/fp32 dtype pick
+ quant = comfy.utils.detect_layer_quantization(sd, "")
+ if quant is not None:
+ return torch.bfloat16, comfy.ops.mixed_precision_ops(quant, torch.bfloat16)
+ load_device = comfy.model_management.get_torch_device()
+ dtype = comfy.model_management.unet_dtype(model_params=-1, supported_dtypes=[torch.bfloat16, torch.float32], weight_dtype=comfy.utils.weight_dtype(sd))
+ manual_cast_dtype = comfy.model_management.unet_manual_cast(dtype, load_device, supported_dtypes=[torch.bfloat16, torch.float32])
+ return dtype, comfy.ops.pick_operations(dtype, manual_cast_dtype, load_device=load_device)
+
+
class ModelPatchLoader:
@classmethod
def INPUT_TYPES(s):
@@ -270,22 +284,27 @@ class ModelPatchLoader:
if torch.count_nonzero(ref_weight) == 0:
config['broken'] = True
model = comfy.ldm.lumina.controlnet.ZImage_Control(device=comfy.model_management.unet_offload_device(), dtype=dtype, operations=comfy.ops.manual_cast, **config)
+ elif 'control_img_in.weight' in sd and 'control_blocks.0.img_mlp.out.weight' in sd: # Qwen Image 2.1 Fun ControlNet
+ dtype, operations = dit_patch_operations(sd)
+ inner_dim = sd["control_img_in.weight"].shape[0]
+ gate_up = sd.get("control_blocks.0.img_mlp.gate_up.weight", None)
+ hidden_dim = gate_up.shape[0] // 2 if gate_up is not None else sd["control_blocks.0.img_mlp.proj.weight"].shape[0]
+ num_blocks = 0
+ while "control_blocks.{}.after_proj.weight".format(num_blocks) in sd:
+ num_blocks += 1
+ model = comfy.ldm.qwen_image21.model.QwenImage21FunControl(
+ num_blocks=num_blocks,
+ control_in_dim=129,
+ inner_dim=inner_dim,
+ attention_head_dim=sd["control_blocks.0.attn.norm_q.weight"].shape[0],
+ mlp_ratio=hidden_dim // inner_dim,
+ fused_mlp=gate_up is not None,
+ operations=operations,
+ device=comfy.model_management.unet_offload_device(),
+ dtype=dtype,
+ )
elif comfy.ldm.minimax.controlnet.is_minimax_h3_fun_state_dict(sd):
- load_device = comfy.model_management.get_torch_device()
- quant = comfy.utils.detect_layer_quantization(sd, "")
- if quant is not None:
- dtype = torch.bfloat16
- operations = comfy.ops.mixed_precision_ops(quant, dtype)
- else:
- dtype = comfy.model_management.unet_dtype(
- model_params=-1,
- supported_dtypes=[torch.bfloat16, torch.float32],
- weight_dtype=comfy.utils.weight_dtype(sd),
- )
- manual_cast_dtype = comfy.model_management.unet_manual_cast(
- dtype, load_device, supported_dtypes=[torch.bfloat16, torch.float32])
- operations = comfy.ops.pick_operations(dtype, manual_cast_dtype, load_device=load_device)
-
+ dtype, operations = dit_patch_operations(sd)
num_blocks = 0
while "control_blocks.{}.after_proj.weight".format(num_blocks) in sd:
num_blocks += 1
@@ -428,13 +447,19 @@ class AnimaLLLiteApply:
return (model_patched,)
+def in_sigma_range(transformer_options, sigma_range):
+ sigma = float(transformer_options["sigmas"].flatten()[0])
+ return sigma_range[1] <= sigma <= sigma_range[0]
+
+
class DiffSynthCnetPatch:
- def __init__(self, model_patch, vae, image, strength, mask=None):
+ def __init__(self, model_patch, vae, image, strength, mask=None, sigma_range=(float("inf"), 0.0)):
self.model_patch = model_patch
self.vae = vae
self.image = image
self.strength = strength
self.mask = mask
+ self.sigma_range = sigma_range
self.encoded_image = model_patch.model.process_input_latent_image(self.encode_latent_cond(image))
self.encoded_image_size = (image.shape[1], image.shape[2])
@@ -451,6 +476,8 @@ class DiffSynthCnetPatch:
return latent_image
def __call__(self, kwargs):
+ if not in_sigma_range(kwargs["transformer_options"], self.sigma_range):
+ return kwargs
x = kwargs.get("x")
img = kwargs.get("img")
block_index = kwargs.get("block_index")
@@ -475,8 +502,9 @@ class DiffSynthCnetPatch:
return [self.model_patch]
class ZImageControlPatch:
- def __init__(self, model_patch, vae, image, strength, inpaint_image=None, mask=None):
+ def __init__(self, model_patch, vae, image, strength, inpaint_image=None, mask=None, sigma_range=(float("inf"), 0.0)):
self.model_patch = model_patch
+ self.sigma_range = sigma_range
self.vae = vae
self.image = image
self.inpaint_image = inpaint_image
@@ -527,6 +555,8 @@ class ZImageControlPatch:
return latent_image
def __call__(self, kwargs):
+ if not in_sigma_range(kwargs["transformer_options"], self.sigma_range):
+ return kwargs
x = kwargs.get("x")
img = kwargs.get("img")
img_input = kwargs.get("img_input")
@@ -597,6 +627,141 @@ class ZImageControlPatch:
def models(self):
return [self.model_patch]
+class QwenImage21FunControlPatch:
+ def __init__(self, model_patch, vae, image, strength, inpaint_image=None, mask=None, sigma_range=(float("inf"), 0.0)):
+ self.model_patch = model_patch
+ self.vae = vae
+ self.image = image
+ self.inpaint_image = inpaint_image
+ self.mask = mask
+ self.strength = strength
+ self.sigma_range = sigma_range
+ self.active = False
+ self.injection_layers = None
+ self.control = None
+ self.stream = None
+ self.pristine = None
+
+ def prepare(self, h, w):
+ # 129 channels per target token: control latents | keep mask | masked-image latents, zeros where not given
+ if self.control is not None and self.control.shape[-2:] == (h, w):
+ return
+ width, height = w * self.vae.spacial_compression_encode(), h * self.vae.spacial_compression_encode()
+ latent_format = comfy.latent_formats.QwenImage21()
+ loaded_models = comfy.model_management.loaded_models(only_currently_used=True)
+ try:
+ control = torch.zeros(1, latent_format.latent_channels, h, w)
+ if self.image is not None:
+ image = comfy.utils.common_upscale(self.image[:1].movedim(-1, 1), width, height, "bicubic", "disabled")
+ control = latent_format.process_in(self.vae.encode(image.movedim(1, -1))).float().cpu()
+ regen = torch.ones(1, 1, height, width)
+ if self.mask is not None:
+ mask = self.mask.reshape(-1, 1, *self.mask.shape[-2:])[:1].float().cpu()
+ regen = (comfy.utils.common_upscale(mask, width, height, "bilinear", "disabled") >= 0.5).float()
+ inpaint = torch.zeros_like(control)
+ if self.inpaint_image is not None:
+ # regenerated pixels at mid-gray, the zero of the VAE's [-1, 1] input
+ image = comfy.utils.common_upscale(self.inpaint_image[:1].movedim(-1, 1), width, height, "bicubic", "disabled").float().cpu()
+ inpaint = latent_format.process_in(self.vae.encode((image * (1 - regen) + 0.5 * regen).movedim(1, -1))).float().cpu()
+ keep = 1 - torch.nn.functional.interpolate(regen, size=(h, w), mode="nearest")
+ finally:
+ comfy.model_management.load_models_gpu(loaded_models)
+ self.control = torch.cat([control, keep, inpaint], dim=1)
+
+ def diffusion_model_wrapper(self, executor, x, timestep, context, ref_latents, image_slots, transformer_options, **kwargs):
+ self.active = in_sigma_range(transformer_options, self.sigma_range)
+ if self.active:
+ with comfy.model_prefetch.pause_malloc_graph():
+ self.prepare(*x.shape[-2:])
+ else:
+ # outside the range the block patches are dropped so the model can use its prefix cache
+ dit = transformer_options.get("patches_replace", {}).get("dit", {})
+ dit = {k: p.previous if isinstance(p, QwenImage21FunControlBlockPatch) and p.control_patch is self else p for k, p in dit.items()}
+ dit = {k: p for k, p in dit.items() if p is not None}
+ transformer_options = {**transformer_options, "patches_replace": {**transformer_options["patches_replace"], "dit": dit}}
+ try:
+ return executor(x, timestep, context, ref_latents, image_slots, transformer_options, **kwargs)
+ finally:
+ self.stream = None
+ self.pristine = None
+
+ def before_block(self, block_index, args):
+ if self.active and block_index == self.injection_layers[0]:
+ # the base block updates its input in place
+ self.pristine = args["img"].clone()
+
+ def after_block(self, block_index, args, out):
+ if not self.active:
+ return out
+ model = self.model_patch.model
+ index = self.injection_layers.index(block_index)
+ if index == 0:
+ self.control = self.control.to(out["img"].device, out["img"].dtype)
+ self.stream = model.init_stream(self.pristine, self.control.flatten(2).transpose(1, 2), args["prefix_len"])
+ self.pristine = None
+ self.stream, skip = model.step(index, self.stream, args["mod"], args["pe"], args["attn_fn"], args["prefix_len"], args["transformer_options"])
+ out["img"].add_(skip, alpha=self.strength)
+ return out
+
+ def to(self, device_or_dtype):
+ if isinstance(device_or_dtype, torch.device):
+ if self.control is not None:
+ self.control = self.control.to(device_or_dtype)
+ self.stream = None
+ return self
+
+ def cleanup(self):
+ self.control = None
+ self.stream = None
+ self.pristine = None
+ self.active = False
+
+ def models(self):
+ return [self.model_patch]
+
+ def register(self, model):
+ # the control blocks pair with every (num_base / num_control)-th base block
+ num_base = len(model.get_model_object("diffusion_model").transformer_blocks)
+ self.injection_layers = list(range(0, num_base, num_base // len(self.model_patch.model.control_blocks)))
+ model.add_wrapper(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, self.diffusion_model_wrapper)
+ blocks_replace = model.model_options.get("transformer_options", {}).get("patches_replace", {}).get("dit", {})
+ for block_index in self.injection_layers:
+ previous = blocks_replace.get(("single_block", block_index))
+ model.set_model_patch_replace(QwenImage21FunControlBlockPatch(self, block_index, previous), "dit", "single_block", block_index)
+
+
+class QwenImage21FunControlBlockPatch:
+ def __init__(self, control_patch, block_index, previous):
+ self.control_patch = control_patch
+ self.block_index = block_index
+ self.previous = previous
+
+ def __call__(self, args, extra_args):
+ # control state stays outside the base block's allocation scope
+ with comfy.model_prefetch.pause_malloc_graph():
+ self.control_patch.before_block(self.block_index, args)
+ out = extra_args["original_block"](args) if self.previous is None else self.previous(args, extra_args)
+ with comfy.model_prefetch.pause_malloc_graph():
+ return self.control_patch.after_block(self.block_index, args, out)
+
+ def to(self, device_or_dtype):
+ self.control_patch.to(device_or_dtype)
+ if hasattr(self.previous, "to"):
+ self.previous = self.previous.to(device_or_dtype)
+ return self
+
+ def cleanup(self):
+ self.control_patch.cleanup()
+ if hasattr(self.previous, "cleanup"):
+ self.previous.cleanup()
+
+ def models(self):
+ models = self.control_patch.models()
+ if hasattr(self.previous, "models"):
+ models += self.previous.models()
+ return models
+
+
class QwenImageDiffsynthControlnet:
@classmethod
def INPUT_TYPES(s):
@@ -606,19 +771,28 @@ class QwenImageDiffsynthControlnet:
"image": ("IMAGE",),
"strength": ("FLOAT", {"default": 1.0, "min": -10.0, "max": 10.0, "step": 0.01}),
},
- "optional": {"mask": ("MASK",)}}
+ "optional": {"mask": ("MASK",),
+ "start_percent": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001, "advanced": True}),
+ "end_percent": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001, "advanced": True})}}
RETURN_TYPES = ("MODEL",)
FUNCTION = "diffsynth_controlnet"
EXPERIMENTAL = True
CATEGORY = "model/patch/qwen"
- def diffsynth_controlnet(self, model, model_patch, vae, image=None, strength=1.0, inpaint_image=None, mask=None):
+ def diffsynth_controlnet(self, model, model_patch, vae, image=None, strength=1.0, inpaint_image=None, mask=None, start_percent=0.0, end_percent=1.0):
+ if strength == 0 or (image is None and inpaint_image is None and mask is None):
+ return (model,)
model_patched = model.clone()
+ model_sampling = model.get_model_object("model_sampling")
+ sigma_range = (float(model_sampling.percent_to_sigma(start_percent)), float(model_sampling.percent_to_sigma(end_percent)))
if image is not None:
image = image[:, :, :, :3]
if inpaint_image is not None:
inpaint_image = inpaint_image[:, :, :, :3]
+ if isinstance(model_patch.model, comfy.ldm.qwen_image21.model.QwenImage21FunControl):
+ QwenImage21FunControlPatch(model_patch, vae, image, strength, inpaint_image=inpaint_image, mask=mask, sigma_range=sigma_range).register(model_patched)
+ return (model_patched,)
if mask is not None:
if mask.ndim == 3:
mask = mask.unsqueeze(1)
@@ -627,11 +801,11 @@ class QwenImageDiffsynthControlnet:
mask = 1.0 - mask
if isinstance(model_patch.model, comfy.ldm.lumina.controlnet.ZImage_Control):
- patch = ZImageControlPatch(model_patch, vae, image, strength, inpaint_image=inpaint_image, mask=mask)
+ patch = ZImageControlPatch(model_patch, vae, image, strength, inpaint_image=inpaint_image, mask=mask, sigma_range=sigma_range)
model_patched.set_model_noise_refiner_patch(patch)
model_patched.set_model_double_block_patch(patch)
else:
- model_patched.set_model_double_block_patch(DiffSynthCnetPatch(model_patch, vae, image, strength, mask))
+ model_patched.set_model_double_block_patch(DiffSynthCnetPatch(model_patch, vae, image, strength, mask, sigma_range=sigma_range))
return (model_patched,)
class ZImageFunControlnet(QwenImageDiffsynthControlnet):
@@ -642,9 +816,13 @@ class ZImageFunControlnet(QwenImageDiffsynthControlnet):
"vae": ("VAE",),
"strength": ("FLOAT", {"default": 1.0, "min": -10.0, "max": 10.0, "step": 0.01}),
},
- "optional": {"image": ("IMAGE",), "inpaint_image": ("IMAGE",), "mask": ("MASK",)}}
+ "optional": {"image": ("IMAGE",), "inpaint_image": ("IMAGE",), "mask": ("MASK",),
+ "start_percent": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001, "advanced": True}),
+ "end_percent": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001, "advanced": True})}}
- CATEGORY = "model/patch/z-image"
+ CATEGORY = "model/patch"
+ SEARCH_ALIASES = ["z-image controlnet", "qwen image 2.1 controlnet", "controlnet union", "fun controlnet"]
+ DESCRIPTION = "Applies a Z-Image or Qwen Image 2.1 Fun ControlNet loaded with Load Model Patch."
class WanUni3CCnetPatch:
def __init__(self, model_patch, render_video, vae, latent_format, strength, sigma_start, sigma_end):
@@ -957,7 +1135,7 @@ NODE_CLASS_MAPPINGS = {
NODE_DISPLAY_NAME_MAPPINGS = {
"ModelPatchLoader": "Load Model Patch",
"QwenImageDiffsynthControlnet": "Apply Qwen Image DiffSynth ControlNet",
- "ZImageFunControlnet": "Apply Z-Image Fun ControlNet",
+ "ZImageFunControlnet": "Apply Fun ControlNet",
"WanUni3CControlnetApply": "Apply Wan Uni3C ControlNet",
"USOStyleReference": "Apply USO Style Reference",
"SUPIRApply": "Apply SUPIR Patch",