The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Add comfy_attention and AttentionTensorContainer to a few models. (#16595)
comfyanonymous8d534945ebd5
diff --git a/comfy/ldm/audio/dit.py b/comfy/ldm/audio/dit.pyindex b0759a240..bdbb43cb5 100644--- a/comfy/ldm/audio/dit.py+++ b/comfy/ldm/audio/dit.py@@ -1,6 +1,6 @@# code adapted from: https://github.com/Stability-AI/stable-audio-tools-from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attentionimport typing as tpimport torch@@ -285,6 +285,7 @@ class Attention(nn.Module):operations=None,):super().__init__()+ self.comfy_attention = ComfyAttention()self.dim = dimself.dim_heads = dim_headsself.causal = causal@@ -430,11 +431,14 @@ class Attention(nn.Module):if self.differential:q, q_diff = q.unbind(dim=1)k, k_diff = k.unbind(dim=1)- out = optimized_attention(q, k, v, h, skip_reshape=True, low_precision_attention=False, transformer_options=transformer_options, **gqa_kwargs)- out_diff = optimized_attention(q_diff, k_diff, v, h, skip_reshape=True, low_precision_attention=False, transformer_options=transformer_options, **gqa_kwargs)+ q, k = AttentionTensorContainer(q), AttentionTensorContainer(k)+ out = optimized_attention(q, k, AttentionTensorContainer(v), h, skip_reshape=True, low_precision_attention=False, preferred_attention=self.comfy_attention, transformer_options=transformer_options, **gqa_kwargs)+ q_diff, k_diff, v = AttentionTensorContainer(q_diff), AttentionTensorContainer(k_diff), AttentionTensorContainer(v)+ out_diff = optimized_attention(q_diff, k_diff, v, h, skip_reshape=True, low_precision_attention=False, preferred_attention=self.comfy_attention, transformer_options=transformer_options, **gqa_kwargs)out = out - out_diffelse:- out = optimized_attention(q, k, v, h, skip_reshape=True, low_precision_attention=False, transformer_options=transformer_options, **gqa_kwargs)+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)+ out = optimized_attention(q, k, v, h, skip_reshape=True, low_precision_attention=False, preferred_attention=self.comfy_attention, transformer_options=transformer_options, **gqa_kwargs)out = self.to_out(out)diff --git a/comfy/ldm/cascade/common.py b/comfy/ldm/cascade/common.pyindex 42ef98c7a..c42a486b7 100644--- a/comfy/ldm/cascade/common.py+++ b/comfy/ldm/cascade/common.py@@ -18,12 +18,13 @@import torchimport torch.nn as nn-from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attentionimport comfy.opsclass OptimizedAttention(nn.Module):def __init__(self, c, nhead, dropout=0.0, dtype=None, device=None, operations=None):super().__init__()+ self.comfy_attention = ComfyAttention()self.heads = nheadself.to_q = operations.Linear(c, c, bias=True, dtype=dtype, device=device)@@ -37,7 +38,8 @@ class OptimizedAttention(nn.Module):k = self.to_k(k)v = self.to_v(v)- out = optimized_attention(q, k, v, self.heads, transformer_options=transformer_options)+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)+ out = optimized_attention(q, k, v, self.heads, preferred_attention=self.comfy_attention, transformer_options=transformer_options)return self.out_proj(out)diff --git a/comfy/ldm/cogvideo/model.py b/comfy/ldm/cogvideo/model.pyindex fb475ed53..a9e09b257 100644--- a/comfy/ldm/cogvideo/model.py+++ b/comfy/ldm/cogvideo/model.py@@ -7,7 +7,7 @@ import torchimport torch.nn as nnimport torch.nn.functional as F-from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attentionimport comfy.patcher_extensionimport comfy.ldm.common_dit@@ -241,6 +241,7 @@ class CogVideoXBlock(nn.Module):eps=1e-5, ff_inner_dim=None, ff_bias=True,device=None, dtype=None, operations=None):super().__init__()+ self.comfy_attention = ComfyAttention()self.dim = dimself.num_heads = num_headsself.head_dim = head_dim@@ -290,12 +291,15 @@ class CogVideoXBlock(nn.Module):k_img = apply_rotary_emb(k_img, image_rotary_emb)q = torch.cat([q[:, :text_seq_length], q_img.transpose(1, 2)], dim=1)k = torch.cat([k[:, :text_seq_length], k_img.transpose(1, 2)], dim=1)+ del q_img, k_img+ q, k, v = AttentionTensorContainer(q.reshape(b, s, n * d)), AttentionTensorContainer(k.reshape(b, s, n * d)), AttentionTensorContainer(v)attn_out = optimized_attention(- q.reshape(b, s, n * d),- k.reshape(b, s, n * d),+ q,+ k,v,heads=self.num_heads,+ preferred_attention=self.comfy_attention,transformer_options=transformer_options,)diff --git a/comfy/ldm/cosmos/blocks.py b/comfy/ldm/cosmos/blocks.pyindex afb43d469..4faae2541 100644--- a/comfy/ldm/cosmos/blocks.py+++ b/comfy/ldm/cosmos/blocks.py@@ -23,7 +23,9 @@ from einops import rearrange, repeatfrom einops.layers.torch import Rearrangefrom torch import nn-from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention+import comfy.model_management+import comfy.quant_opsdef get_normalization(name: str, channels: int, weight_args={}, operations=None):@@ -89,6 +91,7 @@ class Attention(nn.Module):operations=None,) -> None:super().__init__()+ self.comfy_attention = ComfyAttention()self.is_selfattn = context_dim is None # self attention@@ -157,17 +160,20 @@ class Attention(nn.Module):k = self.to_k[1](k)v = self.to_v[1](v)if self.is_selfattn and rope_emb is not None: # only apply to self-attention!- # apply_rotary_pos_emb inlined- q_shape = q.shape- q = q.reshape(*q.shape[:-1], 2, -1).movedim(-2, -1).unsqueeze(-2)- q = rope_emb[..., 0] * q[..., 0] + rope_emb[..., 1] * q[..., 1]- q = q.movedim(-1, -2).reshape(*q_shape).to(x.dtype)-- # apply_rotary_pos_emb inlined- k_shape = k.shape- k = k.reshape(*k.shape[:-1], 2, -1).movedim(-2, -1).unsqueeze(-2)- k = rope_emb[..., 0] * k[..., 0] + rope_emb[..., 1] * k[..., 1]- k = k.movedim(-1, -2).reshape(*k_shape).to(x.dtype)+ if not comfy.model_management.in_training:+ q, k = comfy.quant_ops.ck.apply_rope_split_half(q, k, rope_emb)+ else:+ # apply_rotary_pos_emb inlined+ q_shape = q.shape+ q = q.reshape(*q.shape[:-1], 2, -1).movedim(-2, -1).unsqueeze(-2)+ q = rope_emb[..., 0] * q[..., 0] + rope_emb[..., 1] * q[..., 1]+ q = q.movedim(-1, -2).reshape(*q_shape).to(x.dtype)++ # apply_rotary_pos_emb inlined+ k_shape = k.shape+ k = k.reshape(*k.shape[:-1], 2, -1).movedim(-2, -1).unsqueeze(-2)+ k = rope_emb[..., 0] * k[..., 0] + rope_emb[..., 1] * k[..., 1]+ k = k.movedim(-1, -2).reshape(*k_shape).to(x.dtype)return q, k, vdef forward(@@ -185,7 +191,8 @@ class Attention(nn.Module):context (Optional[Tensor]): The key tensor of shape [B, Mk, K] or use x as context [self attention] if None"""q, k, v = self.cal_qkv(x, context, mask, rope_emb=rope_emb, **kwargs)- out = optimized_attention(q, k, v, self.heads, skip_reshape=True, mask=mask, skip_output_reshape=True, transformer_options=transformer_options)+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)+ out = optimized_attention(q, k, v, self.heads, skip_reshape=True, mask=mask, skip_output_reshape=True, preferred_attention=self.comfy_attention, transformer_options=transformer_options)del q, k, vout = rearrange(out, " b n s c -> s b (n c)")return self.to_out(out)diff --git a/comfy/ldm/cosmos/predict2.py b/comfy/ldm/cosmos/predict2.pyindex 0f4648202..8a5c29b7d 100644--- a/comfy/ldm/cosmos/predict2.py+++ b/comfy/ldm/cosmos/predict2.py@@ -12,7 +12,7 @@ from .position_embedding import VideoRopePosition3DEmb, LearnablePosEmbAxisfrom torchvision import transformsimport comfy.patcher_extension-from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attentionimport comfy.ldm.common_ditimport comfy.opsimport comfy.quant_ops@@ -38,7 +38,7 @@ class GPT2FeedForward(nn.Module):return x-def torch_attention_op(q_B_S_H_D: torch.Tensor, k_B_S_H_D: torch.Tensor, v_B_S_H_D: torch.Tensor, transformer_options: Optional[dict] = {}) -> torch.Tensor:+def torch_attention_op(q_B_S_H_D: torch.Tensor, k_B_S_H_D: torch.Tensor, v_B_S_H_D: torch.Tensor, transformer_options: Optional[dict] = {}, preferred_attention=None) -> torch.Tensor:"""Computes multi-head attention using PyTorch's native implementation.This function provides a PyTorch backend alternative to Transformer Engine's attention operation.@@ -60,12 +60,16 @@ def torch_attention_op(q_B_S_H_D: torch.Tensor, k_B_S_H_D: torch.Tensor, v_B_S_HReturns:Attention output tensor with shape (batch, seq_len, n_heads * head_dim)"""+ if isinstance(q_B_S_H_D, AttentionTensorContainer):+ q_B_S_H_D, k_B_S_H_D, v_B_S_H_D = q_B_S_H_D.take(), k_B_S_H_D.take(), v_B_S_H_D.take()in_q_shape = q_B_S_H_D.shapein_k_shape = k_B_S_H_D.shapeq_B_H_S_D = rearrange(q_B_S_H_D, "b ... h k -> b h ... k").view(in_q_shape[0], in_q_shape[-2], -1, in_q_shape[-1])k_B_H_S_D = rearrange(k_B_S_H_D, "b ... h v -> b h ... v").view(in_k_shape[0], in_k_shape[-2], -1, in_k_shape[-1])v_B_H_S_D = rearrange(v_B_S_H_D, "b ... h v -> b h ... v").view(in_k_shape[0], in_k_shape[-2], -1, in_k_shape[-1])- return optimized_attention(q_B_H_S_D, k_B_H_S_D, v_B_H_S_D, in_q_shape[-2], skip_reshape=True, transformer_options=transformer_options)+ q_B_H_S_D, k_B_H_S_D, v_B_H_S_D = AttentionTensorContainer(q_B_H_S_D), AttentionTensorContainer(k_B_H_S_D), AttentionTensorContainer(v_B_H_S_D)+ del q_B_S_H_D, k_B_S_H_D, v_B_S_H_D+ return optimized_attention(q_B_H_S_D, k_B_H_S_D, v_B_H_S_D, in_q_shape[-2], skip_reshape=True, preferred_attention=preferred_attention, transformer_options=transformer_options)class Attention(nn.Module):@@ -112,6 +116,7 @@ class Attention(nn.Module):operations=None,) -> None:super().__init__()+ self.comfy_attention = ComfyAttention()logging.debug(f"Setting up {self.__class__.__name__}. Query dim is {query_dim}, context_dim is {context_dim} and using "f"{n_heads} heads with a dimension of {head_dim}."@@ -197,7 +202,7 @@ class Attention(nn.Module):return q, k, vdef compute_attention(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, transformer_options: Optional[dict] = {}) -> torch.Tensor:- result = self.attn_op(q, k, v, transformer_options=transformer_options) # [B, S, H, D]+ result = self.attn_op(q, k, v, transformer_options=transformer_options, preferred_attention=self.comfy_attention) # [B, S, H, D]return self.output_dropout(self.output_proj(result))def forward(@@ -213,6 +218,7 @@ class Attention(nn.Module):context (Optional[Tensor]): The key tensor of shape [B, Mk, K] or use x as context [self attention] if None"""q, k, v = self.compute_qkv(x, context, rope_emb=rope_emb, transformer_options=transformer_options)+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)return self.compute_attention(q, k, v, transformer_options=transformer_options)diff --git a/comfy/ldm/genmo/joint_model/asymm_models_joint.py b/comfy/ldm/genmo/joint_model/asymm_models_joint.pyindex 5c1bb4d42..255944894 100644--- a/comfy/ldm/genmo/joint_model/asymm_models_joint.py+++ b/comfy/ldm/genmo/joint_model/asymm_models_joint.py@@ -8,7 +8,7 @@ import torch.nn as nnimport torch.nn.functional as Ffrom einops import rearrange# from flash_attn import flash_attn_varlen_qkvpacked_func-from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attentionfrom .layers import (FeedForward,@@ -68,6 +68,7 @@ class AsymmetricAttention(nn.Module):operations=None,):super().__init__()+ self.comfy_attention = ComfyAttention()self.dim_x = dim_xself.dim_y = dim_yself.num_heads = num_heads@@ -142,13 +143,16 @@ class AsymmetricAttention(nn.Module):k = torch.cat([k_x, k_y[:, :crop_y]], dim=1).transpose(1, 2)v = torch.cat([v_x, v_y[:, :crop_y]], dim=1).transpose(1, 2)+ seq_x, seq_y = q_x.shape[1], q_y.shape[1]+ del q_x, k_x, v_x, q_y, k_y, v_y+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)xy = optimized_attention(q,k,- v, self.num_heads, skip_reshape=True, transformer_options=transformer_options)+ v, self.num_heads, skip_reshape=True, preferred_attention=self.comfy_attention, transformer_options=transformer_options)- x, y = torch.tensor_split(xy, (q_x.shape[1],), dim=1)+ x, y = torch.tensor_split(xy, (seq_x,), dim=1)x = self.proj_x(x)- o = torch.zeros(y.shape[0], q_y.shape[1], y.shape[-1], device=y.device, dtype=y.dtype)+ o = torch.zeros(y.shape[0], seq_y, y.shape[-1], device=y.device, dtype=y.dtype)o[:, :y.shape[1]] = yy = self.proj_y(o)diff --git a/comfy/ldm/genmo/joint_model/utils.py b/comfy/ldm/genmo/joint_model/utils.pyindex 1b399d5d2..a720d758c 100644--- a/comfy/ldm/genmo/joint_model/utils.py+++ b/comfy/ldm/genmo/joint_model/utils.py@@ -4,6 +4,7 @@from typing import Optionalimport torch+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attentionimport torch.nn as nnimport torch.nn.functional as F@@ -51,6 +52,7 @@ class AttentionPool(nn.Module):output_dim (int): Dimensionality of output tokens. Defaults to embed_dim."""super().__init__()+ self.comfy_attention = ComfyAttention()self.num_heads = num_headsself.to_kv = operations.Linear(embed_dim, 2 * embed_dim, device=device, dtype=dtype)self.to_q = operations.Linear(embed_dim, embed_dim, device=device, dtype=dtype)@@ -92,9 +94,9 @@ class AttentionPool(nn.Module):q = q.unsqueeze(2) # (B, H, 1, head_dim)# Compute attention.- x = F.scaled_dot_product_attention(- q, k, v, attn_mask=attn_mask, dropout_p=0.0- ) # (B, H, 1, head_dim)+ del kv+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)+ x = optimized_attention(q, k, v, self.num_heads, mask=attn_mask, skip_reshape=True, skip_output_reshape=True, preferred_attention=self.comfy_attention)# Concatenate heads and run output.x = x.squeeze(2).flatten(1, 2) # (B, D = H * head_dim)diff --git a/comfy/ldm/hunyuan_video/model.py b/comfy/ldm/hunyuan_video/model.pyindex b94cdfa87..12aa98606 100644--- a/comfy/ldm/hunyuan_video/model.py+++ b/comfy/ldm/hunyuan_video/model.py@@ -4,7 +4,7 @@ import torchimport comfy.patcher_extensionimport comfy.ldm.flux.layersimport comfy.ldm.modules.diffusionmodules.mmdit-from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attentionfrom dataclasses import dataclassfrom einops import repeat@@ -63,6 +63,7 @@ class TokenRefinerBlock(nn.Module):operations=None):super().__init__()+ self.comfy_attention = ComfyAttention()self.heads = headsmlp_hidden_dim = hidden_size * 4@@ -88,7 +89,9 @@ class TokenRefinerBlock(nn.Module):norm_x = self.norm1(x)qkv = self.self_attn.qkv(norm_x)q, k, v = qkv.reshape(qkv.shape[0], qkv.shape[1], 3, self.heads, -1).permute(2, 0, 3, 1, 4)- attn = optimized_attention(q, k, v, self.heads, mask=mask, skip_reshape=True, transformer_options=transformer_options)+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)+ del qkv+ attn = optimized_attention(q, k, v, self.heads, mask=mask, skip_reshape=True, preferred_attention=self.comfy_attention, transformer_options=transformer_options)x = x + self.self_attn.proj(attn) * mod1.unsqueeze(1)x = x + self.mlp(self.norm2(x)) * mod2.unsqueeze(1)diff --git a/comfy/ldm/ideogram4/model.py b/comfy/ldm/ideogram4/model.pyindex 12e1a14fb..f9232baf3 100644--- a/comfy/ldm/ideogram4/model.py+++ b/comfy/ldm/ideogram4/model.py@@ -17,7 +17,7 @@ import comfy.opsimport comfy.patcher_extensionimport comfy.quant_opsfrom comfy.ldm.lumina.model import FeedForward-from comfy.ldm.modules.attention import optimized_attention_masked+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention_maskedfrom comfy.text_encoders.llama import precompute_freqs_cis# Per-token role indicators@@ -47,6 +47,7 @@ def _apply_rope_split_half1(x, freqs_cis):class Ideogram4Attention(nn.Module):def __init__(self, hidden_size, num_heads, eps=1e-5, dtype=None, device=None, operations=None):super().__init__()+ self.comfy_attention = ComfyAttention()self.num_heads = num_headsself.head_dim = hidden_size // num_headsself.hidden_size = hidden_size@@ -74,11 +75,12 @@ class Ideogram4Attention(nn.Module):comfy.ops.uncast_bias_weight(self.norm_k, k_scale, None, k_offload_stream)# (B, heads, L, head_dim)- q = q.transpose(1, 2)- k = k.transpose(1, 2)- v = v.transpose(1, 2)+ q = AttentionTensorContainer(q.transpose(1, 2))+ k = AttentionTensorContainer(k.transpose(1, 2))+ v = AttentionTensorContainer(v.transpose(1, 2))+ del qkv- out = optimized_attention_masked(q, k, v, self.num_heads, attn_mask, skip_reshape=True, transformer_options=transformer_options)+ out = optimized_attention_masked(q, k, v, self.num_heads, attn_mask, skip_reshape=True, preferred_attention=self.comfy_attention, transformer_options=transformer_options)return self.o(out)diff --git a/comfy/ldm/joyimage/model.py b/comfy/ldm/joyimage/model.pyindex 9d6951e54..148b95d9a 100644--- a/comfy/ldm/joyimage/model.py+++ b/comfy/ldm/joyimage/model.py@@ -10,7 +10,7 @@ import comfy.ldm.common_ditimport comfy.opsimport comfy.patcher_extensionfrom comfy.ldm.lightricks.model import GELU_approx, PixArtAlphaTextProjection, TimestepEmbedding, Timesteps-from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attentionclass JoyImageModulate(nn.Module):@@ -63,6 +63,7 @@ class JoyImageAttention(nn.Module):):super().__init__()self.num_attention_heads = num_attention_heads+ self.comfy_attention = ComfyAttention()inner_dim = num_attention_heads * attention_head_dimself.img_attn_qkv = operations.Linear(dim, inner_dim * 3, bias=True, dtype=dtype, device=device)@@ -113,12 +114,13 @@ class JoyImageAttention(nn.Module):joint_q = torch.cat([img_q, txt_q], dim=1)joint_k = torch.cat([img_k, txt_k], dim=1)joint_v = torch.cat([img_v, txt_v], dim=1)+ del img_q, img_k, img_v, txt_q, txt_k, txt_v- joint_q = joint_q.flatten(2, 3)- joint_k = joint_k.flatten(2, 3)- joint_v = joint_v.flatten(2, 3)+ joint_q = AttentionTensorContainer(joint_q.flatten(2, 3))+ joint_k = AttentionTensorContainer(joint_k.flatten(2, 3))+ joint_v = AttentionTensorContainer(joint_v.flatten(2, 3))- joint_out = optimized_attention(joint_q, joint_k, joint_v, heads=heads, transformer_options=transformer_options)+ joint_out = optimized_attention(joint_q, joint_k, joint_v, heads=heads, preferred_attention=self.comfy_attention, transformer_options=transformer_options)seq_img = img.shape[1]img_out = joint_out[:, :seq_img, :]diff --git a/comfy/ldm/kandinsky5/model.py b/comfy/ldm/kandinsky5/model.pyindex 1509de2f8..32908b5f6 100644--- a/comfy/ldm/kandinsky5/model.py+++ b/comfy/ldm/kandinsky5/model.py@@ -3,16 +3,20 @@ from torch import nnimport mathimport comfy.ldm.common_dit-from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attentionfrom comfy.ldm.flux.math import apply_rope1from comfy.ldm.flux.layers import EmbedND-def attention(q, k, v, heads, transformer_options={}):+def attention(q, k, v, heads, transformer_options={}, preferred_attention=None):+ if isinstance(q, AttentionTensorContainer):+ q, k, v = q.take(), k.take(), v.take()+ q = AttentionTensorContainer(q.transpose(1, 2))+ k = AttentionTensorContainer(k.transpose(1, 2))+ v = AttentionTensorContainer(v.transpose(1, 2))return optimized_attention(- q.transpose(1, 2),- k.transpose(1, 2),- v.transpose(1, 2),+ q, k, v,heads=heads,+ preferred_attention=preferred_attention,skip_reshape=True,transformer_options=transformer_options)@@ -98,6 +102,7 @@ class Modulation(nn.Module):class SelfAttention(nn.Module):def __init__(self, num_channels, head_dim, operation_settings=None):super().__init__()+ self.comfy_attention = ComfyAttention()assert num_channels % head_dim == 0self.num_heads = num_channels // head_dimself.head_dim = head_dim@@ -120,7 +125,8 @@ class SelfAttention(nn.Module):q = self._compute_qk(x, freqs, self.to_query, self.query_norm)k = self._compute_qk(x, freqs, self.to_key, self.key_norm)v = self.to_value(x).view(*x.shape[:-1], self.num_heads, -1)- out = attention(q, k, v, self.num_heads, transformer_options=transformer_options)+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)+ out = attention(q, k, v, self.num_heads, preferred_attention=self.comfy_attention, transformer_options=transformer_options)return self.out_layer(out)def _forward_chunked(self, x, freqs, transformer_options={}):@@ -135,7 +141,8 @@ class SelfAttention(nn.Module):q = process_chunks(self.to_query, self.query_norm)k = process_chunks(self.to_key, self.key_norm)v = self.to_value(x).view(*x.shape[:-1], self.num_heads, -1)- out = attention(q, k, v, self.num_heads, transformer_options=transformer_options)+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)+ out = attention(q, k, v, self.num_heads, preferred_attention=self.comfy_attention, transformer_options=transformer_options)return self.out_layer(out)def forward(self, x, freqs, transformer_options={}):@@ -154,7 +161,8 @@ class CrossAttention(SelfAttention):def forward(self, x, context, transformer_options={}):q, k, v = self.get_qkv(x, context)- out = attention(self.query_norm(q), self.key_norm(k), v, self.num_heads, transformer_options=transformer_options)+ q, k, v = AttentionTensorContainer(self.query_norm(q)), AttentionTensorContainer(self.key_norm(k)), AttentionTensorContainer(v)+ out = attention(q, k, v, self.num_heads, preferred_attention=self.comfy_attention, transformer_options=transformer_options)return self.out_layer(out)diff --git a/comfy/ldm/krea2/model.py b/comfy/ldm/krea2/model.pyindex 8001812d7..a904e05e0 100644--- a/comfy/ldm/krea2/model.py+++ b/comfy/ldm/krea2/model.py@@ -18,7 +18,7 @@ import comfy.ldm.common_ditimport comfy.utilsfrom comfy.ldm.flux.layers import EmbedND, timestep_embeddingfrom comfy.ldm.flux.math import apply_rope-from comfy.ldm.modules.attention import optimized_attention_masked+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention_maskedclass RMSNorm(nn.Module):@@ -63,6 +63,7 @@ class Attention(nn.Module):def __init__(self, dim: int, heads: int, kvheads: Optional[int] = None, bias: bool = False,device=None, dtype=None, operations=None):super().__init__()+ self.comfy_attention = ComfyAttention()self.heads = headsself.kvheads = kvheads if kvheads is not None else headsself.headdim = dim // self.heads@@ -94,8 +95,9 @@ class Attention(nn.Module):rep = self.heads // self.kvheadsk = k.repeat_interleave(rep, dim=1)v = v.repeat_interleave(rep, dim=1)+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)out = optimized_attention_masked(q, k, v, self.heads, mask=mask, skip_reshape=True,- transformer_options=transformer_options)+ preferred_attention=self.comfy_attention, transformer_options=transformer_options)if "block_index" in transformer_options and "attn1_output_patch" in transformer_patches:for p in transformer_patches["attn1_output_patch"]:diff --git a/comfy/ldm/modules/attention.py b/comfy/ldm/modules/attention.pyindex 7e4f94351..c54415368 100644--- a/comfy/ldm/modules/attention.py+++ b/comfy/ldm/modules/attention.py@@ -956,6 +956,7 @@ def optimized_attention_for_device(device, mask=False, small_input=False):class CrossAttention(nn.Module):def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0., attn_precision=None, dtype=None, device=None, operations=ops):super().__init__()+ self.comfy_attention = ComfyAttention()inner_dim = dim_head * headscontext_dim = default(context_dim, query_dim)self.attn_precision = attn_precision@@ -979,10 +980,11 @@ class CrossAttention(nn.Module):else:v = self.to_v(context)+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)if mask is None:- out = optimized_attention(q, k, v, self.heads, attn_precision=self.attn_precision, transformer_options=transformer_options)+ out = optimized_attention(q, k, v, self.heads, attn_precision=self.attn_precision, preferred_attention=self.comfy_attention, transformer_options=transformer_options)else:- out = optimized_attention_masked(q, k, v, self.heads, mask, attn_precision=self.attn_precision, transformer_options=transformer_options)+ out = optimized_attention_masked(q, k, v, self.heads, mask, attn_precision=self.attn_precision, preferred_attention=self.comfy_attention, transformer_options=transformer_options)return self.to_out(out)diff --git a/comfy/ldm/sensenova/model.py b/comfy/ldm/sensenova/model.pyindex 9dffff301..45a43b8b0 100644--- a/comfy/ldm/sensenova/model.py+++ b/comfy/ldm/sensenova/model.py@@ -6,7 +6,7 @@ import comfy.patcher_extensionimport comfy.utilsfrom comfy.ldm.common_dit import pad_to_patch_sizefrom comfy.ldm.flux.math import apply_rope1-from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attentionfrom comfy.ldm.modules.diffusionmodules.mmdit import TimestepEmbedderfrom .sampling import resolution_noise_scale@@ -190,6 +190,7 @@ class MLP(nn.Module):class Attention(nn.Module):def __init__(self, device=None, dtype=None, operations=None):super().__init__()+ self.comfy_attention = ComfyAttention()self.q_proj = operations.Linear(HIDDEN_SIZE, NUM_HEADS * HEAD_DIM, bias=False, device=device, dtype=dtype)@@ -288,15 +289,16 @@ class Attention(nn.Module):self, hidden_states, rope, attention_mask, transformer_options):query, key, value = self._project(hidden_states, rope, False)+ query = AttentionTensorContainer(query)output = optimized_attention(query,- key,- value,+ AttentionTensorContainer(key),+ AttentionTensorContainer(value),NUM_HEADS,mask=attention_mask,skip_reshape=True,transformer_options=transformer_options,- enable_gqa=True,+ enable_gqa=True, preferred_attention=self.comfy_attention,)return self.o_proj(output), key, value@@ -306,6 +308,7 @@ class Attention(nn.Module):query, key, value = self._project(hidden_states, rope, True)key = torch.cat((prefix_key, key), dim=2)value = torch.cat((prefix_value, value), dim=2)+ query, key, value = AttentionTensorContainer(query), AttentionTensorContainer(key), AttentionTensorContainer(value)output = optimized_attention(query,key,@@ -314,7 +317,7 @@ class Attention(nn.Module):mask=None,skip_reshape=True,transformer_options=transformer_options,- enable_gqa=True,+ enable_gqa=True, preferred_attention=self.comfy_attention,)return self.o_proj_mot_gen(output)diff --git a/comfy/ldm/trellis2/model.py b/comfy/ldm/trellis2/model.pyindex fda7dfd37..352c9c91c 100644--- a/comfy/ldm/trellis2/model.py+++ b/comfy/ldm/trellis2/model.py@@ -5,16 +5,19 @@ import comfy.model_managementimport comfy.opsfrom comfy.ldm.trellis2.vae import SparseTensor, SparseLinear, sparse_cat, VarLenTensorfrom typing import Optional, Tuple, Literal, Union, List-from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attentionfrom comfy.ldm.genmo.joint_model.layers import TimestepEmbedderfrom comfy.ldm.flux.math import apply_rope, apply_rope1def dense_attention(q, k, v, **kwargs):+ if isinstance(q, AttentionTensorContainer):+ q, k, v = q.take(), k.take(), v.take()heads = q.shape[2]q = q.permute(0, 2, 1, 3)k = k.permute(0, 2, 1, 3)v = v.permute(0, 2, 1, 3)+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)out = optimized_attention(q, k, v, heads, skip_output_reshape=True, skip_reshape=True, **kwargs)return out.permute(0, 2, 1, 3)@@ -35,6 +38,8 @@ def _to_padded(t):def sparse_attention(q, k, v, **kwargs):+ if isinstance(q, AttentionTensorContainer):+ q, k, v = q.take(), k.take(), v.take()q_padded, q_valid = _to_padded(q)k_padded, k_valid = _to_padded(k)v_padded, _ = _to_padded(v)@@ -43,6 +48,8 @@ def sparse_attention(q, k, v, **kwargs):dtype=q_padded.dtype, device=q_padded.device)mask.masked_fill_(~k_valid[:, None, None, :], -torch.finfo(q_padded.dtype).max)kwargs["mask"] = mask+ q_padded, k_padded, v_padded = AttentionTensorContainer(q_padded), AttentionTensorContainer(k_padded), AttentionTensorContainer(v_padded)+ del k, vout = dense_attention(q_padded, k_padded, v_padded, **kwargs)if isinstance(q, VarLenTensor):if q_valid is not None:@@ -146,6 +153,7 @@ class SparseMultiHeadAttention(nn.Module):device=None, dtype=None, operations=None):super().__init__()+ self.comfy_attention = ComfyAttention()self.channels = channelsself.head_dim = channels // num_heads@@ -199,18 +207,22 @@ class SparseMultiHeadAttention(nn.Module):if self.qk_rms_norm:q = self.q_rms_norm(q)k = self.k_rms_norm(k)+ del qkvq, k = self.rope(q, k)- h = sparse_attention(q, k, v, transformer_options=transformer_options)+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)+ h = sparse_attention(q, k, v, transformer_options=transformer_options, preferred_attention=self.comfy_attention)else:q = self._linear(self.to_q, x)q = self._reshape_chs(q, (self.num_heads, -1))kv = self._linear(self.to_kv, context)kv = self._fused_pre(kv, num_fused=2)k, v = kv.unbind(dim=-3)+ del kvif self.qk_rms_norm:q = self.q_rms_norm(q)k = self.k_rms_norm(k)- h = sparse_attention(q, k, v, transformer_options=transformer_options)+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)+ h = sparse_attention(q, k, v, transformer_options=transformer_options, preferred_attention=self.comfy_attention)h = self._reshape_chs(h, (-1,))h = self._linear(self.to_out, h)return h@@ -471,6 +483,7 @@ class MultiHeadAttention(nn.Module):device=None, dtype=None, operations=None):super().__init__()+ self.comfy_attention = ComfyAttention()self.channels = channelsself.head_dim = channels // num_heads@@ -505,8 +518,10 @@ class MultiHeadAttention(nn.Module):# phases is [L, head_dim/2, 2, 2]; broadcast to [1, L, 1, ...]# to align with q/k of shape [B, L, H, head_dim].f_cis = comfy.model_management.cast_to(phases, device=q.device).unsqueeze(0).unsqueeze(2)+ del qkvq, k = apply_rope(q, k, f_cis)- h = dense_attention(q, k, v, transformer_options=transformer_options)+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)+ h = dense_attention(q, k, v, transformer_options=transformer_options, preferred_attention=self.comfy_attention)else:Lkv = context.shape[1]q = self.to_q(x)@@ -514,10 +529,12 @@ class MultiHeadAttention(nn.Module):q = q.reshape(B, L, self.num_heads, -1)kv = kv.reshape(B, Lkv, 2, self.num_heads, -1)k, v = kv.unbind(dim=2)+ del kvif self.qk_rms_norm:q = self.q_rms_norm(q)k = self.k_rms_norm(k)- h = dense_attention(q, k, v, transformer_options=transformer_options)+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)+ h = dense_attention(q, k, v, transformer_options=transformer_options, preferred_attention=self.comfy_attention)h = h.reshape(B, L, -1)h = self.to_out(h)return hdiff --git a/comfy/ldm/triposplat/model.py b/comfy/ldm/triposplat/model.pyindex d8a531772..9f4cdb830 100644--- a/comfy/ldm/triposplat/model.py+++ b/comfy/ldm/triposplat/model.py@@ -9,7 +9,7 @@ import torch.nn.functional as Fimport comfy.model_managementimport comfy.patcher_extensionimport comfy.rmsnorm-from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attentionfrom comfy.ldm.flux.math import apply_rope@@ -100,9 +100,13 @@ class PcdAbsolutePositionEmbedder(nn.Module):return out.to(orig_dtype)-def attention(q, k, v, transformer_options=None):+def attention(q, k, v, transformer_options=None, preferred_attention=None):+ if isinstance(q, AttentionTensorContainer):+ q, k, v = q.take(), k.take(), v.take()+ heads = q.shape[2]+ q, k, v = AttentionTensorContainer(q.transpose(1, 2)), AttentionTensorContainer(k.transpose(1, 2)), AttentionTensorContainer(v.transpose(1, 2))# q, k, v: (B, L, heads, dim) -> (B, L, heads, dim). Shared optimized_attention call convention.- out = optimized_attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), heads=q.shape[2],+ out = optimized_attention(q, k, v, heads=heads, preferred_attention=preferred_attention,skip_reshape=True, skip_output_reshape=True, low_precision_attention=False,transformer_options=transformer_options)return out.transpose(1, 2)@@ -127,6 +131,7 @@ class RopeMultiHeadAttention(nn.Module):def __init__(self, channels, num_heads, qkv_bias=True, qk_rms_norm=False, use_rope=False,dtype=None, device=None, operations=None):super().__init__()+ self.comfy_attention = ComfyAttention()self.channels = channelsself.num_heads = num_headsself.head_dim = channels // num_heads@@ -147,7 +152,9 @@ class RopeMultiHeadAttention(nn.Module):if self.qk_rms_norm:q = self.q_norm(q)k = self.k_norm(k)- h = attention(q, k, v, transformer_options) # (B, L, heads, dim)+ del qkv+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)+ h = attention(q, k, v, transformer_options, preferred_attention=self.comfy_attention) # (B, L, heads, dim)return self.out(h.reshape(B, L, C))diff --git a/comfy/ldm/wan/model.py b/comfy/ldm/wan/model.pyindex 1ac306368..0925ac70c 100644--- a/comfy/ldm/wan/model.py+++ b/comfy/ldm/wan/model.py@@ -6,13 +6,14 @@ import torchimport torch.nn as nnfrom einops import rearrange-from comfy.ldm.modules.attention import AttentionTensorContainer, optimized_attention+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attentionfrom comfy.ldm.flux.layers import EmbedNDfrom comfy.ldm.flux.math import apply_rope1, ropeimport comfy.ldm.common_ditimport comfy.model_managementimport comfy.opsimport comfy.patcher_extension+import comfy.quant_opsdef sinusoidal_embedding_1d(dim, position):@@ -40,6 +41,7 @@ class WanSelfAttention(nn.Module):operation_settings={}):assert dim % num_heads == 0super().__init__()+ self.comfy_attention = ComfyAttention()self.dim = dimself.num_heads = num_headsself.head_dim = dim // num_heads@@ -90,6 +92,7 @@ class WanSelfAttention(nn.Module):x = optimized_attention(q, k, v,heads=self.num_heads,+ preferred_attention=self.comfy_attention,transformer_options=transformer_options,)@@ -115,7 +118,7 @@ class WanT2VCrossAttention(WanSelfAttention):v = AttentionTensorContainer(self.v(context))# compute attention- x = optimized_attention(q, k, v, heads=self.num_heads, transformer_options=transformer_options)+ x = optimized_attention(q, k, v, heads=self.num_heads, preferred_attention=self.comfy_attention, transformer_options=transformer_options)x = self.o(x)return x@@ -147,16 +150,16 @@ class WanI2VCrossAttention(WanSelfAttention):# compute query, key, valueq = self.norm_q(self.q(x))- k_img = self.norm_k_img(self.k_img(context_img))- v_img = self.v_img(context_img)+ k_img = AttentionTensorContainer(self.norm_k_img(self.k_img(context_img)))+ v_img = AttentionTensorContainer(self.v_img(context_img))# Sageattn can cause Nans here, don't allow it as there is no speed difference anyway as img attention is tiny.- img_x = optimized_attention(q, k_img, v_img, heads=self.num_heads, transformer_options=transformer_options, low_precision_attention=False)+ img_x = optimized_attention(AttentionTensorContainer(q), k_img, v_img, heads=self.num_heads, preferred_attention=self.comfy_attention, transformer_options=transformer_options, low_precision_attention=False)del k_img, v_img# compute attentionq = AttentionTensorContainer(q)k = AttentionTensorContainer(self.norm_k(self.k(context)))v = AttentionTensorContainer(self.v(context))- x = optimized_attention(q, k, v, heads=self.num_heads, transformer_options=transformer_options)+ x = optimized_attention(q, k, v, heads=self.num_heads, preferred_attention=self.comfy_attention, transformer_options=transformer_options)# outputx = x + img_x@@ -182,6 +185,13 @@ def repeat_e(e, x):return torch.repeat_interleave(e, repeats + 1, dim=1)[:, :x.size(1)]+def modulate(x, norm, shift, scale):+ shift, scale = repeat_e(shift, x), repeat_e(scale, x)+ if comfy.model_management.in_training or not x.is_cuda or x.numel() < 16 * 1024 * 1024:+ return torch.addcmul(shift, norm(x), 1 + scale)+ return comfy.quant_ops.ck.adaln(x, scale, shift, norm.eps)++class WanFeedForward(nn.Sequential):"""[Linear, GELU(tanh), Linear], with the GELU folded into the down-projection."""@@ -257,7 +267,7 @@ class WanAttentionBlock(nn.Module):# self-attentionx = x.contiguous() # otherwise implicit in LayerNormy = self.self_attn(- torch.addcmul(repeat_e(e[0], x), self.norm1(x), 1 + repeat_e(e[1], x)),+ modulate(x, self.norm1, e[0], e[1]),freqs, transformer_options=transformer_options)x = torch.addcmul(x, y, repeat_e(e[2], x))@@ -270,7 +280,7 @@ class WanAttentionBlock(nn.Module):for p in patches["attn2_patch"]:x = p({"x": x, "transformer_options": transformer_options})- y = self.ffn(torch.addcmul(repeat_e(e[3], x), self.norm2(x), 1 + repeat_e(e[4], x)))+ y = self.ffn(modulate(x, self.norm2, e[3], e[4]))x = torch.addcmul(x, y, repeat_e(e[5], x))return x@@ -386,7 +396,7 @@ class Head(nn.Module):else:e = (comfy.model_management.cast_to(self.modulation, dtype=x.dtype, device=x.device).unsqueeze(0) + e.unsqueeze(2)).unbind(2)- x = (self.head(torch.addcmul(repeat_e(e[0], x), self.norm(x), 1 + repeat_e(e[1], x))))+ x = self.head(modulate(x, self.norm, e[0], e[1]))return x@@ -1477,7 +1487,10 @@ class WanT2VCrossAttentionGather(WanSelfAttention):# Handle video spatial structureq = q.reshape(k.shape[0], -1, n, d).transpose(1, 2)- x = optimized_attention(q, k, v, heads=self.num_heads, skip_reshape=True, skip_output_reshape=True, transformer_options=transformer_options)+ q = AttentionTensorContainer(q)+ k = AttentionTensorContainer(k)+ v = AttentionTensorContainer(v)+ x = optimized_attention(q, k, v, heads=self.num_heads, preferred_attention=self.comfy_attention, skip_reshape=True, skip_output_reshape=True, transformer_options=transformer_options)x = x.transpose(1, 2).reshape(b, -1, n * d)x = self.o(x)@@ -1536,7 +1549,7 @@ class WanAttentionBlockAudio(WanAttentionBlock):# self-attentiony = self.self_attn(- torch.addcmul(repeat_e(e[0], x), self.norm1(x), 1 + repeat_e(e[1], x)),+ modulate(x, self.norm1, e[0], e[1]),freqs, transformer_options=transformer_options)x = torch.addcmul(x, y, repeat_e(e[2], x))@@ -1545,7 +1558,7 @@ class WanAttentionBlockAudio(WanAttentionBlock):x = x + self.cross_attn(self.norm3(x), context, context_img_len=context_img_len, transformer_options=transformer_options)if audio is not None:x = self.audio_cross_attn_wrapper(x, audio, transformer_options=transformer_options)- y = self.ffn(torch.addcmul(repeat_e(e[3], x), self.norm2(x), 1 + repeat_e(e[4], x)))+ y = self.ffn(modulate(x, self.norm2, e[3], e[4]))x = torch.addcmul(x, y, repeat_e(e[5], x))return xdiff --git a/comfy/ldm/wan/model_animate.py b/comfy/ldm/wan/model_animate.pyindex 9ebe5694b..91390128b 100644--- a/comfy/ldm/wan/model_animate.py+++ b/comfy/ldm/wan/model_animate.py@@ -5,7 +5,7 @@ from einops import rearrangeimport torch.nn.functional as Fimport mathfrom .model import WanModel, sinusoidal_embedding_1d-from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attentionimport comfy.model_managementclass CausalConv1d(nn.Module):@@ -148,6 +148,7 @@ class FaceBlock(nn.Module):):factory_kwargs = {"device": device, "dtype": dtype}super().__init__()+ self.comfy_attention = ComfyAttention()self.deterministic = Falseself.hidden_size = hidden_size@@ -201,7 +202,11 @@ class FaceBlock(nn.Module):q = rearrange(q, "B (L S) H D -> (B L) S (H D)", L=T_comp)- attn = optimized_attention(q, k, v, heads=self.heads_num)+ q = AttentionTensorContainer(q)+ k = AttentionTensorContainer(k)+ v = AttentionTensorContainer(v)+ del kv+ attn = optimized_attention(q, k, v, heads=self.heads_num, preferred_attention=self.comfy_attention)attn = rearrange(attn, "(B L) S C -> B (L S) C", L=T_comp)diff --git a/comfy/ldm/wan/model_animate2.py b/comfy/ldm/wan/model_animate2.pyindex 8bc55f4e9..cdcc1fbd7 100644--- a/comfy/ldm/wan/model_animate2.py+++ b/comfy/ldm/wan/model_animate2.py@@ -14,9 +14,9 @@ import comfy.model_managementimport comfy.quant_opsimport comfy.utilsfrom comfy.ldm.flux.math import apply_rope1-from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.modules.attention import AttentionTensorContainer, optimized_attention-from .model import WanAttentionBlock, WanModel, WanSelfAttention, repeat_e, sinusoidal_embedding_1d+from .model import WanAttentionBlock, WanModel, WanSelfAttention, modulate, repeat_e, sinusoidal_embedding_1dclass WanAnimate2SelfAttention(WanSelfAttention):@@ -39,7 +39,7 @@ class WanAnimate2SelfAttention(WanSelfAttention):def forward_pose(self, x, freqs, transformer_options={}):b, s, n, d = *x.shape[:2], self.num_heads, self.head_dimq, k, v = self.qkv(x, freqs)- out = optimized_attention(q.reshape(b, s, n * d), k.reshape(b, s, n * d), v.reshape(b, s, n * d), heads=self.num_heads, transformer_options=transformer_options)+ out = optimized_attention(AttentionTensorContainer(q.reshape(b, s, n * d)), AttentionTensorContainer(k.reshape(b, s, n * d)), AttentionTensorContainer(v.reshape(b, s, n * d)), heads=self.num_heads, preferred_attention=self.comfy_attention, transformer_options=transformer_options)return self.o(self._attn1_patch(out, q, k, transformer_options)), k, vdef forward_gen(self, x, freqs, k_pose, v_pose, f_gen, hw, buffers, ref_strength=1.0, transformer_options={}):@@ -50,7 +50,7 @@ class WanAnimate2SelfAttention(WanSelfAttention):v[:, :hw] *= ref_strength # frame 0 is the reference image's slotif k_pose is None: # pose influence windowed out: plain self-attention, no per-frame loop- out = optimized_attention(q.reshape(b, s, n * d), k.reshape(b, s, n * d), v.reshape(b, s, n * d), heads=self.num_heads, transformer_options=transformer_options)+ out = optimized_attention(AttentionTensorContainer(q.reshape(b, s, n * d)), AttentionTensorContainer(k.reshape(b, s, n * d)), AttentionTensorContainer(v.reshape(b, s, n * d)), heads=self.num_heads, preferred_attention=self.comfy_attention, transformer_options=transformer_options)return self.o(self._attn1_patch(out, q, k, transformer_options))# gen half is the same every frame; only the hw-token pose tail is rewritten@@ -66,7 +66,7 @@ class WanAnimate2SelfAttention(WanSelfAttention):kbuf[:, s:] = k_pose[:, (j - 1) * hw:j * hw]vbuf[:, s:] = v_pose[:, (j - 1) * hw:j * hw]kk, vv = kbuf, vbuf- out[:, j * hw:(j + 1) * hw] = optimized_attention(q_j, kk.reshape(b, kk.shape[1], n * d), vv.reshape(b, kk.shape[1], n * d), heads=self.num_heads, transformer_options=transformer_options)+ out[:, j * hw:(j + 1) * hw] = optimized_attention(AttentionTensorContainer(q_j), AttentionTensorContainer(kk.reshape(b, kk.shape[1], n * d)), AttentionTensorContainer(vv.reshape(b, kk.shape[1], n * d)), heads=self.num_heads, preferred_attention=self.comfy_attention, transformer_options=transformer_options)return self.o(self._attn1_patch(out, q, k, transformer_options))@@ -85,13 +85,13 @@ class WanAnimate2Block(WanAttentionBlock):x = x + self.cross_attn(self.norm3(x), context, context_img_len=context_img_len, transformer_options=transformer_options)for p in transformer_options.get("patches", {}).get("attn2_patch", []):x = p({"x": x, "transformer_options": transformer_options})- y = self.ffn(torch.addcmul(repeat_e(e[3], x), self.norm2(x), 1 + repeat_e(e[4], x)))+ y = self.ffn(modulate(x, self.norm2, e[3], e[4]))return torch.addcmul(x, y, repeat_e(e[5], x))def forward_pose(self, x, e, freqs, context, context_img_len=257, transformer_options={}):e = self._modulation(e, x)x = x.contiguous()- y, k, v = self.self_attn.forward_pose(torch.addcmul(repeat_e(e[0], x), self.norm1(x), 1 + repeat_e(e[1], x)), freqs, transformer_options=transformer_options)+ y, k, v = self.self_attn.forward_pose(modulate(x, self.norm1, e[0], e[1]), freqs, transformer_options=transformer_options)x = torch.addcmul(x, y, repeat_e(e[2], x))del yreturn self._cross_attn_ffn(x, e, context, context_img_len, transformer_options), k, v@@ -99,12 +99,12 @@ class WanAnimate2Block(WanAttentionBlock):def kv_from_input(self, x_pose, e, freqs, transformer_options={}):e = self._modulation(e, x_pose)x_pose = x_pose.contiguous()- return self.self_attn.kv(torch.addcmul(repeat_e(e[0], x_pose), self.norm1(x_pose), 1 + repeat_e(e[1], x_pose)), freqs)+ return self.self_attn.kv(modulate(x_pose, self.norm1, e[0], e[1]), freqs)def forward_gen(self, x, e, freqs, context, k_pose, v_pose, f_gen, hw, buffers, ref_strength=1.0, context_img_len=257, transformer_options={}):e = self._modulation(e, x)x = x.contiguous()- y = self.self_attn.forward_gen(torch.addcmul(repeat_e(e[0], x), self.norm1(x), 1 + repeat_e(e[1], x)), freqs, k_pose, v_pose, f_gen, hw, buffers, ref_strength=ref_strength, transformer_options=transformer_options)+ y = self.self_attn.forward_gen(modulate(x, self.norm1, e[0], e[1]), freqs, k_pose, v_pose, f_gen, hw, buffers, ref_strength=ref_strength, transformer_options=transformer_options)x = torch.addcmul(x, y, repeat_e(e[2], x))del yreturn self._cross_attn_ffn(x, e, context, context_img_len, transformer_options)diff --git a/comfy/ldm/wan/model_multitalk.py b/comfy/ldm/wan/model_multitalk.pyindex c9dd98c4d..951beb2e8 100644--- a/comfy/ldm/wan/model_multitalk.py+++ b/comfy/ldm/wan/model_multitalk.py@@ -1,7 +1,7 @@import torchfrom einops import rearrange, repeatimport comfy-from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attentiondef calculate_x_ref_attn_map(visual_q, ref_k, ref_target_masks, split_num=8):@@ -171,6 +171,7 @@ class SingleStreamAttention(torch.nn.Module):) -> None:super().__init__()self.dim = dim+ self.comfy_attention = ComfyAttention()self.encoder_hidden_states_dim = encoder_hidden_states_dimself.num_heads = num_headsself.head_dim = dim // num_heads@@ -203,11 +204,11 @@ class SingleStreamAttention(torch.nn.Module):encoder_k, encoder_v = kv.view(B * N_t, encoder_hidden_states.shape[1], 2, self.num_heads, self.head_dim).unbind(2)#print("q.shape", q.shape) #torch.Size([21, 1024, 40, 128])- x = optimized_attention(- q.transpose(1, 2),- encoder_k.transpose(1, 2),- encoder_v.transpose(1, 2),- heads=self.num_heads, skip_reshape=True, skip_output_reshape=True).transpose(1, 2)+ q = AttentionTensorContainer(q.transpose(1, 2))+ encoder_k = AttentionTensorContainer(encoder_k.transpose(1, 2))+ encoder_v = AttentionTensorContainer(encoder_v.transpose(1, 2))+ del kv+ x = optimized_attention(q, encoder_k, encoder_v, heads=self.num_heads, preferred_attention=self.comfy_attention, skip_reshape=True, skip_output_reshape=True).transpose(1, 2)# linear transformx = self.proj(x.reshape(B * N_t, S, self.dim))@@ -324,11 +325,11 @@ class SingleStreamMultiAttention(SingleStreamAttention):encoder_k = rearrange(encoder_k, "B H M K -> B M H K")encoder_v = rearrange(encoder_v, "B H M K -> B M H K")- x = optimized_attention(- q.transpose(1, 2),- encoder_k.transpose(1, 2),- encoder_v.transpose(1, 2),- heads=self.num_heads, skip_reshape=True, skip_output_reshape=True).transpose(1, 2)+ q = AttentionTensorContainer(q.transpose(1, 2))+ encoder_k = AttentionTensorContainer(encoder_k.transpose(1, 2))+ encoder_v = AttentionTensorContainer(encoder_v.transpose(1, 2))+ del encoder_kv+ x = optimized_attention(q, encoder_k, encoder_v, heads=self.num_heads, preferred_attention=self.comfy_attention, skip_reshape=True, skip_output_reshape=True).transpose(1, 2)# Linear projectionx = x.reshape(B, N, C)diff --git a/comfy/ldm/wan/model_wandancer.py b/comfy/ldm/wan/model_wandancer.pyindex aeec1d725..6de415c84 100644--- a/comfy/ldm/wan/model_wandancer.py+++ b/comfy/ldm/wan/model_wandancer.py@@ -1,7 +1,7 @@import torchimport torch.nn as nnimport comfy-from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attentionfrom comfy.ldm.flux.math import apply_rope1from comfy.ldm.flux.layers import EmbedND@@ -12,6 +12,7 @@ class MusicSelfAttention(nn.Module):def __init__(self, dim, num_heads, device=None, dtype=None, operations=None):assert dim % num_heads == 0super().__init__()+ self.comfy_attention = ComfyAttention()self.embed_dim = dimself.num_heads = num_headsself.head_dim = dim // num_heads@@ -30,11 +31,13 @@ class MusicSelfAttention(nn.Module):k = self.k_proj(x).view(b, s, n, d)k = apply_rope1(k, freqs)+ q = AttentionTensorContainer(q.view(b, s, n * d))+ k = AttentionTensorContainer(k.view(b, s, n * d))+ v = AttentionTensorContainer(self.v_proj(x).view(b, s, n * d))x = optimized_attention(- q.view(b, s, n * d),- k.view(b, s, n * d),- self.v_proj(x).view(b, s, n * d),+ q, k, v,heads=self.num_heads,+ preferred_attention=self.comfy_attention,)return self.out_proj(x)diff --git a/tests-unit/comfy_test/test_sensenova.py b/tests-unit/comfy_test/test_sensenova.pyindex 4e2398e41..505080e27 100644--- a/tests-unit/comfy_test/test_sensenova.py+++ b/tests-unit/comfy_test/test_sensenova.py@@ -584,6 +584,7 @@ def test_sensenova_prefix_mask_matches_attention_dtype(monkeypatch):attention = SimpleNamespace(_project=lambda hidden_states, rope, generation: (query, key, value),o_proj=lambda output: output,+ comfy_attention=sensenova_model.ComfyAttention(),)mask = torch.zeros(1, 1, 3, 3, dtype=torch.bfloat16)