The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Support the Anima model. (#12012)
comfyanonymousabe2ec26a61f
diff --git a/comfy/ldm/anima/model.py b/comfy/ldm/anima/model.pynew file mode 100644index 000000000..2e6ed58fa--- /dev/null+++ b/comfy/ldm/anima/model.py@@ -0,0 +1,202 @@+from comfy.ldm.cosmos.predict2 import MiniTrainDIT+import torch+from torch import nn+import torch.nn.functional as F+++def rotate_half(x):+ x1 = x[..., : x.shape[-1] // 2]+ x2 = x[..., x.shape[-1] // 2 :]+ return torch.cat((-x2, x1), dim=-1)+++def apply_rotary_pos_emb(x, cos, sin, unsqueeze_dim=1):+ cos = cos.unsqueeze(unsqueeze_dim)+ sin = sin.unsqueeze(unsqueeze_dim)+ x_embed = (x * cos) + (rotate_half(x) * sin)+ return x_embed+++class RotaryEmbedding(nn.Module):+ def __init__(self, head_dim):+ super().__init__()+ self.rope_theta = 10000+ inv_freq = 1.0 / (self.rope_theta ** (torch.arange(0, head_dim, 2, dtype=torch.int64).to(dtype=torch.float) / head_dim))+ self.register_buffer("inv_freq", inv_freq, persistent=False)++ @torch.no_grad()+ def forward(self, x, position_ids):+ inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1).to(x.device)+ position_ids_expanded = position_ids[:, None, :].float()++ device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu"+ with torch.autocast(device_type=device_type, enabled=False): # Force float32+ freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)+ emb = torch.cat((freqs, freqs), dim=-1)+ cos = emb.cos()+ sin = emb.sin()++ return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)+++class Attention(nn.Module):+ def __init__(self, query_dim, context_dim, n_heads, head_dim, device=None, dtype=None, operations=None):+ super().__init__()++ inner_dim = head_dim * n_heads+ self.n_heads = n_heads+ self.head_dim = head_dim+ self.query_dim = query_dim+ self.context_dim = context_dim++ self.q_proj = operations.Linear(query_dim, inner_dim, bias=False, device=device, dtype=dtype)+ self.q_norm = operations.RMSNorm(self.head_dim, eps=1e-6, device=device, dtype=dtype)++ self.k_proj = operations.Linear(context_dim, inner_dim, bias=False, device=device, dtype=dtype)+ self.k_norm = operations.RMSNorm(self.head_dim, eps=1e-6, device=device, dtype=dtype)++ self.v_proj = operations.Linear(context_dim, inner_dim, bias=False, device=device, dtype=dtype)++ self.o_proj = operations.Linear(inner_dim, query_dim, bias=False, device=device, dtype=dtype)++ def forward(self, x, mask=None, context=None, position_embeddings=None, position_embeddings_context=None):+ context = x if context is None else context+ input_shape = x.shape[:-1]+ q_shape = (*input_shape, self.n_heads, self.head_dim)+ context_shape = context.shape[:-1]+ kv_shape = (*context_shape, self.n_heads, self.head_dim)++ query_states = self.q_norm(self.q_proj(x).view(q_shape)).transpose(1, 2)+ key_states = self.k_norm(self.k_proj(context).view(kv_shape)).transpose(1, 2)+ value_states = self.v_proj(context).view(kv_shape).transpose(1, 2)++ if position_embeddings is not None:+ assert position_embeddings_context is not None+ cos, sin = position_embeddings+ query_states = apply_rotary_pos_emb(query_states, cos, sin)+ cos, sin = position_embeddings_context+ key_states = apply_rotary_pos_emb(key_states, cos, sin)++ attn_output = F.scaled_dot_product_attention(query_states, key_states, value_states, attn_mask=mask)++ attn_output = attn_output.transpose(1, 2).reshape(*input_shape, -1).contiguous()+ attn_output = self.o_proj(attn_output)+ return attn_output++ def init_weights(self):+ torch.nn.init.zeros_(self.o_proj.weight)+++class TransformerBlock(nn.Module):+ def __init__(self, source_dim, model_dim, num_heads=16, mlp_ratio=4.0, use_self_attn=False, layer_norm=False, device=None, dtype=None, operations=None):+ super().__init__()+ self.use_self_attn = use_self_attn++ if self.use_self_attn:+ self.norm_self_attn = operations.LayerNorm(model_dim, device=device, dtype=dtype) if layer_norm else operations.RMSNorm(model_dim, eps=1e-6, device=device, dtype=dtype)+ self.self_attn = Attention(+ query_dim=model_dim,+ context_dim=model_dim,+ n_heads=num_heads,+ head_dim=model_dim//num_heads,+ device=device,+ dtype=dtype,+ operations=operations,+ )++ self.norm_cross_attn = operations.LayerNorm(model_dim, device=device, dtype=dtype) if layer_norm else operations.RMSNorm(model_dim, eps=1e-6, device=device, dtype=dtype)+ self.cross_attn = Attention(+ query_dim=model_dim,+ context_dim=source_dim,+ n_heads=num_heads,+ head_dim=model_dim//num_heads,+ device=device,+ dtype=dtype,+ operations=operations,+ )++ self.norm_mlp = operations.LayerNorm(model_dim, device=device, dtype=dtype) if layer_norm else operations.RMSNorm(model_dim, eps=1e-6, device=device, dtype=dtype)+ self.mlp = nn.Sequential(+ operations.Linear(model_dim, int(model_dim * mlp_ratio), device=device, dtype=dtype),+ nn.GELU(),+ operations.Linear(int(model_dim * mlp_ratio), model_dim, device=device, dtype=dtype)+ )++ def forward(self, x, context, target_attention_mask=None, source_attention_mask=None, position_embeddings=None, position_embeddings_context=None):+ if self.use_self_attn:+ normed = self.norm_self_attn(x)+ attn_out = self.self_attn(normed, mask=target_attention_mask, position_embeddings=position_embeddings, position_embeddings_context=position_embeddings)+ x = x + attn_out++ normed = self.norm_cross_attn(x)+ attn_out = self.cross_attn(normed, mask=source_attention_mask, context=context, position_embeddings=position_embeddings, position_embeddings_context=position_embeddings_context)+ x = x + attn_out++ x = x + self.mlp(self.norm_mlp(x))+ return x++ def init_weights(self):+ torch.nn.init.zeros_(self.mlp[2].weight)+ self.cross_attn.init_weights()+++class LLMAdapter(nn.Module):+ def __init__(+ self,+ source_dim=1024,+ target_dim=1024,+ model_dim=1024,+ num_layers=6,+ num_heads=16,+ use_self_attn=True,+ layer_norm=False,+ device=None,+ dtype=None,+ operations=None,+ ):+ super().__init__()++ self.embed = operations.Embedding(32128, target_dim, device=device, dtype=dtype)+ if model_dim != target_dim:+ self.in_proj = operations.Linear(target_dim, model_dim, device=device, dtype=dtype)+ else:+ self.in_proj = nn.Identity()+ self.rotary_emb = RotaryEmbedding(model_dim//num_heads)+ self.blocks = nn.ModuleList([+ TransformerBlock(source_dim, model_dim, num_heads=num_heads, use_self_attn=use_self_attn, layer_norm=layer_norm, device=device, dtype=dtype, operations=operations) for _ in range(num_layers)+ ])+ self.out_proj = operations.Linear(model_dim, target_dim, device=device, dtype=dtype)+ self.norm = operations.RMSNorm(target_dim, eps=1e-6, device=device, dtype=dtype)++ def forward(self, source_hidden_states, target_input_ids, target_attention_mask=None, source_attention_mask=None):+ if target_attention_mask is not None:+ target_attention_mask = target_attention_mask.to(torch.bool)+ if target_attention_mask.ndim == 2:+ target_attention_mask = target_attention_mask.unsqueeze(1).unsqueeze(1)++ if source_attention_mask is not None:+ source_attention_mask = source_attention_mask.to(torch.bool)+ if source_attention_mask.ndim == 2:+ source_attention_mask = source_attention_mask.unsqueeze(1).unsqueeze(1)++ x = self.in_proj(self.embed(target_input_ids))+ context = source_hidden_states+ position_ids = torch.arange(x.shape[1], device=x.device).unsqueeze(0)+ position_ids_context = torch.arange(context.shape[1], device=x.device).unsqueeze(0)+ position_embeddings = self.rotary_emb(x, position_ids)+ position_embeddings_context = self.rotary_emb(x, position_ids_context)+ for block in self.blocks:+ x = block(x, context, target_attention_mask=target_attention_mask, source_attention_mask=source_attention_mask, position_embeddings=position_embeddings, position_embeddings_context=position_embeddings_context)+ return self.norm(self.out_proj(x))+++class Anima(MiniTrainDIT):+ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)+ self.llm_adapter = LLMAdapter(device=kwargs.get("device"), dtype=kwargs.get("dtype"), operations=kwargs.get("operations"))++ def preprocess_text_embeds(self, text_embeds, text_ids):+ if text_ids is not None:+ return self.llm_adapter(text_embeds, text_ids)+ else:+ return text_embedsdiff --git a/comfy/model_base.py b/comfy/model_base.pyindex 28ba2643e..1d57562cc 100644--- a/comfy/model_base.py+++ b/comfy/model_base.py@@ -49,6 +49,7 @@ import comfy.ldm.ace.modelimport comfy.ldm.omnigen.omnigen2import comfy.ldm.qwen_image.modelimport comfy.ldm.kandinsky5.model+import comfy.ldm.anima.modelimport comfy.model_managementimport comfy.patcher_extension@@ -1147,6 +1148,27 @@ class CosmosPredict2(BaseModel):sigma = (sigma / (sigma + 1))return latent_image / (1.0 - sigma)+class Anima(BaseModel):+ def __init__(self, model_config, model_type=ModelType.FLOW, device=None):+ super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.anima.model.Anima)++ def extra_conds(self, **kwargs):+ out = super().extra_conds(**kwargs)+ cross_attn = kwargs.get("cross_attn", None)+ t5xxl_ids = kwargs.get("t5xxl_ids", None)+ t5xxl_weights = kwargs.get("t5xxl_weights", None)+ device = kwargs["device"]+ if cross_attn is not None:+ if t5xxl_ids is not None:+ cross_attn = self.diffusion_model.preprocess_text_embeds(cross_attn.to(device=device, dtype=self.get_dtype()), t5xxl_ids.unsqueeze(0).to(device=device))+ if t5xxl_weights is not None:+ cross_attn *= t5xxl_weights.unsqueeze(0).unsqueeze(-1).to(cross_attn)++ if cross_attn.shape[1] < 512:+ cross_attn = torch.nn.functional.pad(cross_attn, (0, 0, 0, 512 - cross_attn.shape[1]))+ out['c_crossattn'] = comfy.conds.CONDRegular(cross_attn)+ return out+class Lumina2(BaseModel):def __init__(self, model_config, model_type=ModelType.FLOW, device=None):super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.lumina.model.NextDiT)diff --git a/comfy/model_detection.py b/comfy/model_detection.pyindex dad206a2f..b29a033cc 100644--- a/comfy/model_detection.py+++ b/comfy/model_detection.py@@ -550,6 +550,8 @@ def detect_unet_config(state_dict, key_prefix, metadata=None):if '{}blocks.0.mlp.layer1.weight'.format(key_prefix) in state_dict_keys: # Cosmos predict2dit_config = {}dit_config["image_model"] = "cosmos_predict2"+ if "{}llm_adapter.blocks.0.cross_attn.q_proj.weight".format(key_prefix) in state_dict_keys:+ dit_config["image_model"] = "anima"dit_config["max_img_h"] = 240dit_config["max_img_w"] = 240dit_config["max_frames"] = 128diff --git a/comfy/sd.py b/comfy/sd.pyindex 77700dfd3..f7f6a44a0 100644--- a/comfy/sd.py+++ b/comfy/sd.py@@ -57,6 +57,7 @@ import comfy.text_encoders.ovisimport comfy.text_encoders.kandinsky5import comfy.text_encoders.jina_clip_2import comfy.text_encoders.newbie+import comfy.text_encoders.animaimport comfy.model_patcherimport comfy.lora@@ -1048,6 +1049,7 @@ class TEModel(Enum):GEMMA_3_12B = 18JINA_CLIP_2 = 19QWEN3_8B = 20+ QWEN3_06B = 21def detect_te_model(sd):@@ -1093,6 +1095,8 @@ def detect_te_model(sd):return TEModel.QWEN3_2Belif weight.shape[0] == 4096:return TEModel.QWEN3_8B+ elif weight.shape[0] == 1024:+ return TEModel.QWEN3_06Bif weight.shape[0] == 5120:if "model.layers.39.post_attention_layernorm.weight" in sd:return TEModel.MISTRAL3_24B@@ -1233,6 +1237,9 @@ def load_text_encoder_state_dicts(state_dicts=[], embedding_directory=None, clipelif te_model == TEModel.JINA_CLIP_2:clip_target.clip = comfy.text_encoders.jina_clip_2.JinaClip2TextModelWrapperclip_target.tokenizer = comfy.text_encoders.jina_clip_2.JinaClip2TokenizerWrapper+ elif te_model == TEModel.QWEN3_06B:+ clip_target.clip = comfy.text_encoders.anima.te(**llama_detect(clip_data))+ clip_target.tokenizer = comfy.text_encoders.anima.AnimaTokenizerelse:# clip_lif clip_type == CLIPType.SD3:diff --git a/comfy/supported_models.py b/comfy/supported_models.pyindex c8a7f6efb..70abebf46 100644--- a/comfy/supported_models.py+++ b/comfy/supported_models.py@@ -23,6 +23,7 @@ import comfy.text_encoders.qwen_imageimport comfy.text_encoders.hunyuan_imageimport comfy.text_encoders.kandinsky5import comfy.text_encoders.z_image+import comfy.text_encoders.animafrom . import supported_models_basefrom . import latent_formats@@ -992,6 +993,36 @@ class CosmosT2IPredict2(supported_models_base.BASE):t5_detect = comfy.text_encoders.sd3_clip.t5_xxl_detect(state_dict, "{}t5xxl.transformer.".format(pref))return supported_models_base.ClipTarget(comfy.text_encoders.cosmos.CosmosT5Tokenizer, comfy.text_encoders.cosmos.te(**t5_detect))+class Anima(supported_models_base.BASE):+ unet_config = {+ "image_model": "anima",+ }++ sampling_settings = {+ "multiplier": 1.0,+ "shift": 3.0,+ }++ unet_extra_config = {}+ latent_format = latent_formats.Wan21++ memory_usage_factor = 1.0++ supported_inference_dtypes = [torch.bfloat16, torch.float32]++ def __init__(self, unet_config):+ super().__init__(unet_config)+ self.memory_usage_factor = (unet_config.get("model_channels", 2048) / 2048) * 0.95++ def get_model(self, state_dict, prefix="", device=None):+ out = model_base.Anima(self, device=device)+ return out++ def clip_target(self, state_dict={}):+ pref = self.text_encoder_key_prefix[0]+ detect = comfy.text_encoders.hunyuan_video.llama_detect(state_dict, "{}qwen3_06b.transformer.".format(pref))+ return supported_models_base.ClipTarget(comfy.text_encoders.anima.AnimaTokenizer, comfy.text_encoders.anima.te(**detect))+class CosmosI2VPredict2(CosmosT2IPredict2):unet_config = {"image_model": "cosmos_predict2",@@ -1551,6 +1582,6 @@ class Kandinsky5Image(Kandinsky5):return supported_models_base.ClipTarget(comfy.text_encoders.kandinsky5.Kandinsky5TokenizerImage, comfy.text_encoders.kandinsky5.te(**hunyuan_detect))-models = [LotusD, Stable_Zero123, SD15_instructpix2pix, SD15, SD20, SD21UnclipL, SD21UnclipH, SDXL_instructpix2pix, SDXLRefiner, SDXL, SSD1B, KOALA_700M, KOALA_1B, Segmind_Vega, SD_X4Upscaler, Stable_Cascade_C, Stable_Cascade_B, SV3D_u, SV3D_p, SD3, StableAudio, AuraFlow, PixArtAlpha, PixArtSigma, HunyuanDiT, HunyuanDiT1, FluxInpaint, Flux, FluxSchnell, GenmoMochi, LTXV, LTXAV, HunyuanVideo15_SR_Distilled, HunyuanVideo15, HunyuanImage21Refiner, HunyuanImage21, HunyuanVideoSkyreelsI2V, HunyuanVideoI2V, HunyuanVideo, CosmosT2V, CosmosI2V, CosmosT2IPredict2, CosmosI2VPredict2, ZImage, Lumina2, WAN22_T2V, WAN21_T2V, WAN21_I2V, WAN21_FunControl2V, WAN21_Vace, WAN21_Camera, WAN22_Camera, WAN22_S2V, WAN21_HuMo, WAN22_Animate, Hunyuan3Dv2mini, Hunyuan3Dv2, Hunyuan3Dv2_1, HiDream, Chroma, ChromaRadiance, ACEStep, Omnigen2, QwenImage, Flux2, Kandinsky5Image, Kandinsky5]+models = [LotusD, Stable_Zero123, SD15_instructpix2pix, SD15, SD20, SD21UnclipL, SD21UnclipH, SDXL_instructpix2pix, SDXLRefiner, SDXL, SSD1B, KOALA_700M, KOALA_1B, Segmind_Vega, SD_X4Upscaler, Stable_Cascade_C, Stable_Cascade_B, SV3D_u, SV3D_p, SD3, StableAudio, AuraFlow, PixArtAlpha, PixArtSigma, HunyuanDiT, HunyuanDiT1, FluxInpaint, Flux, FluxSchnell, GenmoMochi, LTXV, LTXAV, HunyuanVideo15_SR_Distilled, HunyuanVideo15, HunyuanImage21Refiner, HunyuanImage21, HunyuanVideoSkyreelsI2V, HunyuanVideoI2V, HunyuanVideo, CosmosT2V, CosmosI2V, CosmosT2IPredict2, CosmosI2VPredict2, ZImage, Lumina2, WAN22_T2V, WAN21_T2V, WAN21_I2V, WAN21_FunControl2V, WAN21_Vace, WAN21_Camera, WAN22_Camera, WAN22_S2V, WAN21_HuMo, WAN22_Animate, Hunyuan3Dv2mini, Hunyuan3Dv2, Hunyuan3Dv2_1, HiDream, Chroma, ChromaRadiance, ACEStep, Omnigen2, QwenImage, Flux2, Kandinsky5Image, Kandinsky5, Anima]models += [SVD_img2vid]diff --git a/comfy/text_encoders/anima.py b/comfy/text_encoders/anima.pynew file mode 100644index 000000000..41f95bcb6--- /dev/null+++ b/comfy/text_encoders/anima.py@@ -0,0 +1,61 @@+from transformers import Qwen2Tokenizer, T5TokenizerFast+import comfy.text_encoders.llama+from comfy import sd1_clip+import os+import torch+++class Qwen3Tokenizer(sd1_clip.SDTokenizer):+ def __init__(self, embedding_directory=None, tokenizer_data={}):+ tokenizer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "qwen25_tokenizer")+ super().__init__(tokenizer_path, pad_with_end=False, embedding_size=1024, embedding_key='qwen3_06b', tokenizer_class=Qwen2Tokenizer, has_start_token=False, has_end_token=False, pad_to_max_length=False, max_length=99999999, min_length=1, pad_token=151643, tokenizer_data=tokenizer_data)++class T5XXLTokenizer(sd1_clip.SDTokenizer):+ def __init__(self, embedding_directory=None, tokenizer_data={}):+ tokenizer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "t5_tokenizer")+ super().__init__(tokenizer_path, embedding_directory=embedding_directory, pad_with_end=False, embedding_size=4096, embedding_key='t5xxl', tokenizer_class=T5TokenizerFast, has_start_token=False, pad_to_max_length=False, max_length=99999999, min_length=1, tokenizer_data=tokenizer_data)++class AnimaTokenizer:+ def __init__(self, embedding_directory=None, tokenizer_data={}):+ self.qwen3_06b = Qwen3Tokenizer(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data)+ self.t5xxl = T5XXLTokenizer(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data)++ def tokenize_with_weights(self, text:str, return_word_ids=False, **kwargs):+ out = {}+ qwen_ids = self.qwen3_06b.tokenize_with_weights(text, return_word_ids, **kwargs)+ out["qwen3_06b"] = [[(token, 1.0) for token, _ in inner_list] for inner_list in qwen_ids] # Set weights to 1.0+ out["t5xxl"] = self.t5xxl.tokenize_with_weights(text, return_word_ids, **kwargs)+ return out++ def untokenize(self, token_weight_pair):+ return self.t5xxl.untokenize(token_weight_pair)++ def state_dict(self):+ return {}+++class Qwen3_06BModel(sd1_clip.SDClipModel):+ def __init__(self, device="cpu", layer="last", layer_idx=None, dtype=None, attention_mask=True, model_options={}):+ super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config={}, dtype=dtype, special_tokens={"pad": 151643}, layer_norm_hidden_state=False, model_class=comfy.text_encoders.llama.Qwen3_06B, enable_attention_masks=attention_mask, return_attention_masks=attention_mask, model_options=model_options)+++class AnimaTEModel(sd1_clip.SD1ClipModel):+ def __init__(self, device="cpu", dtype=None, model_options={}):+ super().__init__(device=device, dtype=dtype, name="qwen3_06b", clip_model=Qwen3_06BModel, model_options=model_options)++ def encode_token_weights(self, token_weight_pairs):+ out = super().encode_token_weights(token_weight_pairs)+ out[2]["t5xxl_ids"] = torch.tensor(list(map(lambda a: a[0], token_weight_pairs["t5xxl"][0])), dtype=torch.int)+ out[2]["t5xxl_weights"] = torch.tensor(list(map(lambda a: a[1], token_weight_pairs["t5xxl"][0])))+ return out++def te(dtype_llama=None, llama_quantization_metadata=None):+ class AnimaTEModel_(AnimaTEModel):+ def __init__(self, device="cpu", dtype=None, model_options={}):+ if dtype_llama is not None:+ dtype = dtype_llama+ if llama_quantization_metadata is not None:+ model_options = model_options.copy()+ model_options["quantization_metadata"] = llama_quantization_metadata+ super().__init__(device=device, dtype=dtype, model_options=model_options)+ return AnimaTEModel_