The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Make built in lora training work on anima. (#12402)
comfyanonymous76a7fa96dbdc
diff --git a/comfy/ldm/anima/model.py b/comfy/ldm/anima/model.pyindex 2e6ed58fa..6fb51c4a4 100644--- a/comfy/ldm/anima/model.py+++ b/comfy/ldm/anima/model.py@@ -195,8 +195,20 @@ class Anima(MiniTrainDIT):super().__init__(*args, **kwargs)self.llm_adapter = LLMAdapter(device=kwargs.get("device"), dtype=kwargs.get("dtype"), operations=kwargs.get("operations"))- def preprocess_text_embeds(self, text_embeds, text_ids):+ def preprocess_text_embeds(self, text_embeds, text_ids, t5xxl_weights=None):if text_ids is not None:- return self.llm_adapter(text_embeds, text_ids)+ out = self.llm_adapter(text_embeds, text_ids)+ if t5xxl_weights is not None:+ out = out * t5xxl_weights++ if out.shape[1] < 512:+ out = torch.nn.functional.pad(out, (0, 0, 0, 512 - out.shape[1]))+ return outelse:return text_embeds++ def forward(self, x, timesteps, context, **kwargs):+ t5xxl_ids = kwargs.pop("t5xxl_ids", None)+ if t5xxl_ids is not None:+ context = self.preprocess_text_embeds(context, t5xxl_ids, t5xxl_weights=kwargs.pop("t5xxl_weights", None))+ return super().forward(x, timesteps, context, **kwargs)diff --git a/comfy/model_base.py b/comfy/model_base.pyindex 858789b30..4a74cb1ce 100644--- a/comfy/model_base.py+++ b/comfy/model_base.py@@ -1160,12 +1160,16 @@ class Anima(BaseModel):device = kwargs["device"]if cross_attn is not None:if t5xxl_ids is not None:- cross_attn = self.diffusion_model.preprocess_text_embeds(cross_attn.to(device=device, dtype=self.get_dtype()), t5xxl_ids.unsqueeze(0).to(device=device))if t5xxl_weights is not None:- cross_attn *= t5xxl_weights.unsqueeze(0).unsqueeze(-1).to(cross_attn)+ t5xxl_weights = t5xxl_weights.unsqueeze(0).unsqueeze(-1).to(cross_attn)+ t5xxl_ids = t5xxl_ids.unsqueeze(0)++ if torch.is_inference_mode_enabled(): # if not we are training+ cross_attn = self.diffusion_model.preprocess_text_embeds(cross_attn.to(device=device, dtype=self.get_dtype()), t5xxl_ids.to(device=device), t5xxl_weights=t5xxl_weights.to(device=device, dtype=self.get_dtype()))+ else:+ out['t5xxl_ids'] = comfy.conds.CONDRegular(t5xxl_ids)+ out['t5xxl_weights'] = comfy.conds.CONDRegular(t5xxl_weights)- if cross_attn.shape[1] < 512:- cross_attn = torch.nn.functional.pad(cross_attn, (0, 0, 0, 512 - cross_attn.shape[1]))out['c_crossattn'] = comfy.conds.CONDRegular(cross_attn)return out