The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Support anima lllite control models. (#14954)
comfyanonymous0f42ba514631
diff --git a/comfy/ldm/anima/lllite.py b/comfy/ldm/anima/lllite.pynew file mode 100644index 000000000..5c950ec89--- /dev/null+++ b/comfy/ldm/anima/lllite.py@@ -0,0 +1,278 @@+import re++import torch+from torch import nn+import torch.nn.functional as F++import comfy.ops+import comfy.utils+++MODULE_PATTERN = re.compile(r"lllite_dit_blocks_(\d+)_(self_attn_[qkv]_proj|cross_attn_q_proj|mlp_layer1)$")+++def _group_norm(channels, device=None, dtype=None, operations=None):+ groups = 8+ while groups > 1 and channels % groups != 0:+ groups //= 2+ return operations.GroupNorm(groups, channels, device=device, dtype=dtype)+++class AnimaLLLiteResBlock(nn.Module):+ def __init__(self, channels, device=None, dtype=None, operations=None):+ super().__init__()+ self.norm1 = _group_norm(channels, device=device, dtype=dtype, operations=operations)+ self.conv1 = operations.Conv2d(channels, channels, kernel_size=3, padding=1, device=device, dtype=dtype)+ self.norm2 = _group_norm(channels, device=device, dtype=dtype, operations=operations)+ self.conv2 = operations.Conv2d(channels, channels, kernel_size=3, padding=1, device=device, dtype=dtype)++ def forward(self, x):+ h = self.conv1(F.silu(self.norm1(x)))+ h = self.conv2(F.silu(self.norm2(h)))+ return x + h+++class AnimaLLLiteASPP(nn.Module):+ def __init__(self, channels, dilations, device=None, dtype=None, operations=None):+ super().__init__()+ branches = []+ for dilation in dilations:+ if dilation == 1:+ conv = operations.Conv2d(channels, channels, kernel_size=1, device=device, dtype=dtype)+ else:+ conv = operations.Conv2d(channels, channels, kernel_size=3, padding=dilation, dilation=dilation, device=device, dtype=dtype)+ branches.append(nn.Sequential(conv, _group_norm(channels, device=device, dtype=dtype, operations=operations), nn.SiLU()))+ self.branches = nn.ModuleList(branches)+ self.global_pool = nn.AdaptiveAvgPool2d(1)+ self.global_conv = nn.Sequential(+ operations.Conv2d(channels, channels, kernel_size=1, device=device, dtype=dtype),+ _group_norm(channels, device=device, dtype=dtype, operations=operations),+ nn.SiLU(),+ )+ self.proj = nn.Sequential(+ operations.Conv2d(channels * (len(dilations) + 1), channels, kernel_size=1, device=device, dtype=dtype),+ _group_norm(channels, device=device, dtype=dtype, operations=operations),+ nn.SiLU(),+ )++ def forward(self, x):+ height, width = x.shape[-2:]+ outputs = [branch(x) for branch in self.branches]+ pooled = self.global_conv(self.global_pool(x))+ outputs.append(F.interpolate(pooled, size=(height, width), mode="bilinear", align_corners=False))+ return self.proj(torch.cat(outputs, dim=1))+++class AnimaLLLiteConditioning(nn.Module):+ def __init__(self, cond_in_channels, cond_dim, cond_emb_dim, cond_resblocks, aspp_dilations, device=None, dtype=None, operations=None):+ super().__init__()+ half_dim = cond_dim // 2+ self.conv1 = operations.Conv2d(cond_in_channels, half_dim, kernel_size=4, stride=4, device=device, dtype=dtype)+ self.norm1 = _group_norm(half_dim, device=device, dtype=dtype, operations=operations)+ self.conv2 = operations.Conv2d(half_dim, half_dim, kernel_size=3, padding=1, device=device, dtype=dtype)+ self.norm2 = _group_norm(half_dim, device=device, dtype=dtype, operations=operations)+ self.conv3 = operations.Conv2d(half_dim, cond_dim, kernel_size=4, stride=4, device=device, dtype=dtype)+ self.norm3 = _group_norm(cond_dim, device=device, dtype=dtype, operations=operations)+ self.resblocks = nn.ModuleList([+ AnimaLLLiteResBlock(cond_dim, device=device, dtype=dtype, operations=operations)+ for _ in range(cond_resblocks)+ ])+ self.aspp = AnimaLLLiteASPP(cond_dim, aspp_dilations, device=device, dtype=dtype, operations=operations) if aspp_dilations else None+ self.proj = operations.Conv2d(cond_dim, cond_emb_dim, kernel_size=1, device=device, dtype=dtype)+ self.out_norm = operations.LayerNorm(cond_emb_dim, device=device, dtype=dtype)++ def forward(self, x):+ x = F.silu(self.norm1(self.conv1(x)))+ x = F.silu(self.norm2(self.conv2(x)))+ x = F.silu(self.norm3(self.conv3(x)))+ for block in self.resblocks:+ x = block(x)+ if self.aspp is not None:+ x = self.aspp(x)+ x = self.proj(x).flatten(2).transpose(1, 2).contiguous()+ return self.out_norm(x)+++class AnimaLLLiteModule(nn.Module):+ def __init__(self, in_dim, cond_emb_dim, mlp_dim, device=None, dtype=None, operations=None):+ super().__init__()+ self.down = operations.Linear(in_dim, mlp_dim, device=device, dtype=dtype)+ self.mid = operations.Linear(mlp_dim + cond_emb_dim, mlp_dim, device=device, dtype=dtype)+ self.cond_to_film = operations.Linear(cond_emb_dim, 2 * mlp_dim, device=device, dtype=dtype)+ self.up = operations.Linear(mlp_dim, in_dim, device=device, dtype=dtype)+ self.depth_embed = nn.Parameter(torch.empty(cond_emb_dim, device=device, dtype=dtype), requires_grad=False)++ def forward(self, x, cond_emb, strength):+ original_shape = x.shape+ if x.ndim == 5:+ x = x.flatten(1, 3)++ if x.shape[0] != cond_emb.shape[0]:+ if x.shape[0] % cond_emb.shape[0] != 0:+ raise ValueError(f"Anima LLLite batch mismatch: model input batch {x.shape[0]}, control batch {cond_emb.shape[0]}")+ cond_emb = cond_emb.repeat(x.shape[0] // cond_emb.shape[0], 1, 1)+ if x.shape[1] != cond_emb.shape[1]:+ raise ValueError(f"Anima LLLite sequence mismatch: model input has {x.shape[1]} tokens, control has {cond_emb.shape[1]}")++ cond_local = cond_emb + comfy.ops.cast_to_input(self.depth_embed, cond_emb)+ hidden = F.silu(self.down(x))+ gamma, beta = self.cond_to_film(cond_local).chunk(2, dim=-1)+ hidden = self.mid(torch.cat((cond_local, hidden), dim=-1))+ hidden = F.silu(hidden * (1 + gamma) + beta)+ x = x + self.up(hidden) * strength++ if len(original_shape) == 5:+ x = x.reshape(original_shape)+ return x+++class AnimaLLLite(nn.Module):+ def __init__(self, state_dict, metadata, device=None, dtype=None, operations=None):+ super().__init__()+ metadata = metadata or {}+ version = metadata.get("lllite.version", "2")+ if version != "2":+ raise ValueError(f"Unsupported Anima LLLite version {version!r}; only named-key v2 checkpoints are supported")++ module_names = sorted({key.split(".", 1)[0] for key in state_dict if key.startswith("lllite_dit_blocks_")})+ if not module_names:+ raise ValueError("Anima LLLite checkpoint has no lllite_dit_blocks_* modules")++ cond_in_channels = state_dict["lllite_conditioning1.conv1.weight"].shape[1]+ cond_dim = state_dict["lllite_conditioning1.conv3.weight"].shape[0]+ cond_emb_dim = state_dict["lllite_conditioning1.proj.weight"].shape[0]+ resblock_ids = {int(key.split(".")[2]) for key in state_dict if key.startswith("lllite_conditioning1.resblocks.")}+ cond_resblocks = max(resblock_ids) + 1 if resblock_ids else 0+ use_aspp = any(key.startswith("lllite_conditioning1.aspp.") for key in state_dict)+ dilation_string = metadata.get("lllite.aspp_dilations", "1,2,4,8")+ aspp_dilations = tuple(int(value) for value in dilation_string.split(",") if value.strip()) if use_aspp else ()++ self.cond_in_channels = cond_in_channels+ self.inpaint_masked_input = metadata.get("lllite.inpaint_masked_input", "false").lower() == "true"+ self.lllite_conditioning1 = AnimaLLLiteConditioning(+ cond_in_channels, cond_dim, cond_emb_dim, cond_resblocks, aspp_dilations,+ device=device, dtype=dtype, operations=operations,+ )++ self.module_names = set()+ self.block_count = 0+ self.model_dim = None+ for name in module_names:+ match = MODULE_PATTERN.fullmatch(name)+ if match is None:+ raise ValueError(f"Unsupported Anima LLLite module name: {name}")+ down_shape = state_dict[f"{name}.down.weight"].shape+ mlp_dim, in_dim = down_shape+ module_cond_dim = state_dict[f"{name}.cond_to_film.weight"].shape[1]+ if module_cond_dim != cond_emb_dim:+ raise ValueError(f"Anima LLLite conditioning dimension mismatch in {name}: {module_cond_dim} != {cond_emb_dim}")+ if self.model_dim is None:+ self.model_dim = in_dim+ elif self.model_dim != in_dim:+ raise ValueError(f"Anima LLLite model dimension mismatch in {name}: {in_dim} != {self.model_dim}")+ self.add_module(name, AnimaLLLiteModule(in_dim, cond_emb_dim, mlp_dim, device=device, dtype=dtype, operations=operations))+ self.module_names.add(name)+ self.block_count = max(self.block_count, int(match.group(1)) + 1)++ def encode_conditioning(self, image):+ return self.lllite_conditioning1(image)++ def apply(self, x, cond_emb, block_index, target, strength):+ name = f"lllite_dit_blocks_{block_index}_{target}"+ if name not in self.module_names:+ return x+ return self.get_submodule(name)(x, cond_emb, strength)+++class AnimaLLLitePatch:+ def __init__(self, model_patch, image, mask, strength, sigma_start, sigma_end):+ self.model_patch = model_patch+ self.image = image+ self.mask = mask+ self.strength = strength+ self.sigma_start = sigma_start+ self.sigma_end = sigma_end++ def __call__(self, args):+ x = args["x"]+ transformer_options = args["transformer_options"]+ if self.strength == 0.0:+ return args+ sigmas = transformer_options.get("sigmas")+ if sigmas is not None:+ sigma = float(sigmas.max().item())+ if not self.sigma_end <= sigma <= self.sigma_start:+ return args+ if x.shape[2] != 1:+ raise ValueError(f"Anima LLLite only supports T=1, got T={x.shape[2]}")++ target_height = x.shape[-2] * 8+ target_width = x.shape[-1] * 8+ image = comfy.utils.common_upscale(+ self.image.movedim(-1, 1), target_width, target_height, "bicubic", crop="center"+ ).clamp(0.0, 1.0)+ image = image.to(device=x.device, dtype=x.dtype) * 2.0 - 1.0++ if self.model_patch.model.cond_in_channels == 4:+ mask = self.mask+ if mask.ndim == 3:+ mask = mask.unsqueeze(1)+ if mask.ndim != 4 or mask.shape[1] != 1:+ raise ValueError(f"Anima LLLite mask must have one channel, got shape {tuple(mask.shape)}")+ mask = comfy.utils.common_upscale(+ mask.float(), target_width, target_height, "nearest-exact", crop="center"+ )+ if mask.shape[0] != image.shape[0]:+ if image.shape[0] % mask.shape[0] != 0:+ raise ValueError(+ f"Anima LLLite mask batch {mask.shape[0]} cannot be broadcast to image batch {image.shape[0]}"+ )+ mask = mask.repeat(image.shape[0] // mask.shape[0], 1, 1, 1)+ mask = (mask >= 0.5).to(device=x.device, dtype=x.dtype)+ if self.model_patch.model.inpaint_masked_input:+ image = image * (mask < 0.5).to(image.dtype)+ image = torch.cat((image, mask * 2.0 - 1.0), dim=1)++ cond_emb = self.model_patch.model.encode_conditioning(image)+ transformer_options["model_patch_data"][self] = cond_emb+ return args++ def to(self, device_or_dtype):+ return self++ def models(self):+ return [self.model_patch]+++class AnimaLLLiteAttentionPatch:+ def __init__(self, patch, targets):+ self.patch = patch+ self.targets = targets++ def __call__(self, q, k, v, pe=None, attn_mask=None, extra_options=None):+ cond_emb = extra_options["model_patch_data"].get(self.patch)+ if cond_emb is None:+ return {"q": q, "k": k, "v": v, "pe": pe, "attn_mask": attn_mask}++ block_index = extra_options["block_index"]+ values = {"q": q, "k": k, "v": v}+ for value_name, target in self.targets.items():+ values[value_name] = self.patch.model_patch.model.apply(+ values[value_name], cond_emb, block_index, target, self.patch.strength+ )++ return {"q": values["q"], "k": values["k"], "v": values["v"], "pe": pe, "attn_mask": attn_mask}+++class AnimaLLLiteMLPPatch:+ def __init__(self, patch):+ self.patch = patch++ def __call__(self, args):+ cond_emb = args["transformer_options"]["model_patch_data"].get(self.patch)+ if cond_emb is None:+ return args+ args["x"] = self.patch.model_patch.model.apply(+ args["x"], cond_emb, args["transformer_options"]["block_index"], "mlp_layer1", self.patch.strength+ )+ return argsdiff --git a/comfy/ldm/cosmos/predict2.py b/comfy/ldm/cosmos/predict2.pyindex 371296e21..d391d50b1 100644--- a/comfy/ldm/cosmos/predict2.py+++ b/comfy/ldm/cosmos/predict2.py@@ -149,11 +149,29 @@ class Attention(nn.Module):x: torch.Tensor,context: Optional[torch.Tensor] = None,rope_emb: Optional[torch.Tensor] = None,+ transformer_options: Optional[dict] = {},) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:- q = self.q_proj(x)context = x if context is None else context- k = self.k_proj(context)- v = self.v_proj(context)+ q_input = x+ k_input = context+ v_input = context++ transformer_patches = transformer_options.get("patches", {})+ patch_name = "attn1_patch" if self.is_selfattn else "attn2_patch"+ if patch_name in transformer_patches:+ extra_options = transformer_options.copy()+ extra_options["n_heads"] = self.n_heads+ extra_options["dim_head"] = self.head_dim+ for patch in transformer_patches[patch_name]:+ out = patch(q_input, k_input, v_input, pe=rope_emb, attn_mask=None, extra_options=extra_options)+ q_input = out.get("q", q_input)+ k_input = out.get("k", k_input)+ v_input = out.get("v", v_input)+ rope_emb = out.get("pe", rope_emb)++ q = self.q_proj(q_input)+ k = self.k_proj(k_input)+ v = self.v_proj(v_input)q, k, v = map(lambda t: rearrange(t, "b ... (h d) -> b ... h d", h=self.n_heads, d=self.head_dim),(q, k, v),@@ -194,7 +212,7 @@ class Attention(nn.Module):x (Tensor): The query tensor of shape [B, Mq, K]context (Optional[Tensor]): The key tensor of shape [B, Mk, K] or use x as context [self attention] if None"""- q, k, v = self.compute_qkv(x, context, rope_emb=rope_emb)+ q, k, v = self.compute_qkv(x, context, rope_emb=rope_emb, transformer_options=transformer_options)return self.compute_attention(q, k, v, transformer_options=transformer_options)@@ -561,8 +579,14 @@ class Block(nn.Module):self.layer_norm_mlp,scale_mlp_B_T_1_1_D,shift_mlp_B_T_1_1_D,- )- result_B_T_H_W_D = self.mlp(normalized_x_B_T_H_W_D.to(compute_dtype))+ ).to(compute_dtype)+ patches = transformer_options.get("patches", {})+ if "mlp_patch" in patches:+ args = {"x": normalized_x_B_T_H_W_D, "transformer_options": transformer_options}+ for patch in patches["mlp_patch"]:+ args = patch(args)+ normalized_x_B_T_H_W_D = args["x"]+ result_B_T_H_W_D = self.mlp(normalized_x_B_T_H_W_D)x_B_T_H_W_D = torch.addcmul(x_B_T_H_W_D, gate_mlp_B_T_1_1_D.to(residual_dtype), result_B_T_H_W_D.to(residual_dtype))return x_B_T_H_W_D@@ -869,11 +893,22 @@ class MiniTrainDIT(nn.Module):x_B_T_H_W_D.shape == extra_pos_emb_B_T_H_W_D_or_T_H_W_B_D.shape), f"{x_B_T_H_W_D.shape} != {extra_pos_emb_B_T_H_W_D_or_T_H_W_B_D.shape}"+ transformer_options = kwargs.get("transformer_options", {})+ patches = transformer_options.get("patches", {})+ if "post_input" in patches:+ transformer_options = transformer_options.copy()+ transformer_options["model_patch_data"] = {}++ if "post_input" in patches:+ for patch in patches["post_input"]:+ out = patch({"img": x_B_T_H_W_D, "x": x_B_C_T_H_W, "transformer_options": transformer_options})+ x_B_T_H_W_D = out["img"]+block_kwargs = {"rope_emb_L_1_1_D": rope_emb_L_1_1_D.unsqueeze(1).unsqueeze(0),"adaln_lora_B_T_3D": adaln_lora_B_T_3D,"extra_per_block_pos_emb": extra_pos_emb_B_T_H_W_D_or_T_H_W_B_D,- "transformer_options": kwargs.get("transformer_options", {}),+ "transformer_options": transformer_options,}# The residual stream for this model has large values. To make fp16 compute_dtype work, we keep the residual stream@@ -883,7 +918,8 @@ class MiniTrainDIT(nn.Module):if x_B_T_H_W_D.dtype == torch.float16:x_B_T_H_W_D = x_B_T_H_W_D.float()- for block in self.blocks:+ for block_index, block in enumerate(self.blocks):+ transformer_options["block_index"] = block_indexx_B_T_H_W_D = block(x_B_T_H_W_D,t_embedding_B_T_D,diff --git a/comfy_extras/nodes_model_patch.py b/comfy_extras/nodes_model_patch.pyindex 3f785c8b5..0935af09d 100644--- a/comfy_extras/nodes_model_patch.py+++ b/comfy_extras/nodes_model_patch.py@@ -8,6 +8,7 @@ import comfy.ldm.common_ditimport comfy.latent_formatsimport comfy.ldm.lumina.controlnetimport comfy.ldm.supir.supir_modules+import comfy.ldm.anima.lllitefrom comfy.ldm.wan.model_multitalk import WanMultiTalkAttentionBlock, MultiTalkAudioProjModelfrom comfy_api.latest import iofrom comfy.ldm.supir.supir_patch import SUPIRPatch@@ -236,10 +237,12 @@ class ModelPatchLoader:def load_model_patch(self, name):model_patch_path = folder_paths.get_full_path_or_raise("model_patches", name)- sd = comfy.utils.load_torch_file(model_patch_path, safe_load=True)+ sd, metadata = comfy.utils.load_torch_file(model_patch_path, safe_load=True, return_metadata=True)dtype = comfy.utils.weight_dtype(sd)- if 'controlnet_blocks.0.y_rms.weight' in sd:+ if 'lllite_conditioning1.conv1.weight' in sd:+ model = comfy.ldm.anima.lllite.AnimaLLLite(sd, metadata, device=comfy.model_management.unet_offload_device(), dtype=dtype, operations=comfy.ops.manual_cast)+ elif 'controlnet_blocks.0.y_rms.weight' in sd:additional_in_dim = sd["img_in.weight"].shape[1] - 64model = QwenImageBlockWiseControlNet(additional_in_dim=additional_in_dim, device=comfy.model_management.unet_offload_device(), dtype=dtype, operations=comfy.ops.manual_cast)elif 'feature_embedder.mid_layer_norm.bias' in sd:@@ -296,6 +299,50 @@ class ModelPatchLoader:return (model_patcher,)+class AnimaLLLiteApply:+ @classmethod+ def INPUT_TYPES(s):+ return {"required": {"model": ("MODEL",),+ "model_patch": ("MODEL_PATCH",),+ "image": ("IMAGE",),+ "strength": ("FLOAT", {"default": 1.0, "min": -10.0, "max": 10.0, "step": 0.01}),+ "start_percent": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001}),+ "end_percent": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001}),+ },+ "optional": {"mask": ("MASK",),+ }}+ RETURN_TYPES = ("MODEL",)+ FUNCTION = "apply_patch"+ EXPERIMENTAL = True++ CATEGORY = "model_patches/anima"++ def apply_patch(self, model, model_patch, image, strength, start_percent, end_percent, mask=None):+ image = image[..., :3]++ if model_patch.model.cond_in_channels == 4 and mask is None:+ mask = torch.zeros_like(image[..., 0])+ elif model_patch.model.cond_in_channels != 4:+ mask = None++ model_sampling = model.get_model_object("model_sampling")+ sigma_start = float(model_sampling.percent_to_sigma(start_percent))+ sigma_end = float(model_sampling.percent_to_sigma(end_percent))+ patch = comfy.ldm.anima.lllite.AnimaLLLitePatch(model_patch, image, mask, strength, sigma_start, sigma_end)+ model_patched = model.clone()+ model_patched.set_model_post_input_patch(patch)+ model_patched.set_model_attn1_patch(comfy.ldm.anima.lllite.AnimaLLLiteAttentionPatch(+ patch,+ {"q": "self_attn_q_proj", "k": "self_attn_k_proj", "v": "self_attn_v_proj"},+ ))+ model_patched.set_model_attn2_patch(comfy.ldm.anima.lllite.AnimaLLLiteAttentionPatch(+ patch,+ {"q": "cross_attn_q_proj"},+ ))+ model_patched.set_model_patch(comfy.ldm.anima.lllite.AnimaLLLiteMLPPatch(patch), "mlp_patch")+ return (model_patched,)++class DiffSynthCnetPatch:def __init__(self, model_patch, vae, image, strength, mask=None):self.model_patch = model_patch@@ -674,6 +721,7 @@ NODE_CLASS_MAPPINGS = {"ZImageFunControlnet": ZImageFunControlnet,"USOStyleReference": USOStyleReference,"SUPIRApply": SUPIRApply,+ "AnimaLLLiteApply": AnimaLLLiteApply,}NODE_DISPLAY_NAME_MAPPINGS = {@@ -682,4 +730,5 @@ NODE_DISPLAY_NAME_MAPPINGS = {"ZImageFunControlnet": "Apply Z-Image Fun ControlNet","USOStyleReference": "Apply USO Style Reference","SUPIRApply": "Apply SUPIR Patch",+ "AnimaLLLiteApply": "Apply Anima LLLite",}