Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Add comfy_attention and AttentionTensorContainer to a few models. (#16595)

comfyanonymous8d534945ebd5
diff --git a/comfy/ldm/audio/dit.py b/comfy/ldm/audio/dit.py
index b0759a240..bdbb43cb5 100644
--- a/comfy/ldm/audio/dit.py
+++ b/comfy/ldm/audio/dit.py
@@ -1,6 +1,6 @@
# code adapted from: https://github.com/Stability-AI/stable-audio-tools
-from comfy.ldm.modules.attention import optimized_attention
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention
import typing as tp
import torch
@@ -285,6 +285,7 @@ class Attention(nn.Module):
operations=None,
):
super().__init__()
+ self.comfy_attention = ComfyAttention()
self.dim = dim
self.dim_heads = dim_heads
self.causal = causal
@@ -430,11 +431,14 @@ class Attention(nn.Module):
if self.differential:
q, q_diff = q.unbind(dim=1)
k, k_diff = k.unbind(dim=1)
- out = optimized_attention(q, k, v, h, skip_reshape=True, low_precision_attention=False, transformer_options=transformer_options, **gqa_kwargs)
- out_diff = optimized_attention(q_diff, k_diff, v, h, skip_reshape=True, low_precision_attention=False, transformer_options=transformer_options, **gqa_kwargs)
+ q, k = AttentionTensorContainer(q), AttentionTensorContainer(k)
+ out = optimized_attention(q, k, AttentionTensorContainer(v), h, skip_reshape=True, low_precision_attention=False, preferred_attention=self.comfy_attention, transformer_options=transformer_options, **gqa_kwargs)
+ q_diff, k_diff, v = AttentionTensorContainer(q_diff), AttentionTensorContainer(k_diff), AttentionTensorContainer(v)
+ out_diff = optimized_attention(q_diff, k_diff, v, h, skip_reshape=True, low_precision_attention=False, preferred_attention=self.comfy_attention, transformer_options=transformer_options, **gqa_kwargs)
out = out - out_diff
else:
- out = optimized_attention(q, k, v, h, skip_reshape=True, low_precision_attention=False, transformer_options=transformer_options, **gqa_kwargs)
+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)
+ out = optimized_attention(q, k, v, h, skip_reshape=True, low_precision_attention=False, preferred_attention=self.comfy_attention, transformer_options=transformer_options, **gqa_kwargs)
out = self.to_out(out)
diff --git a/comfy/ldm/cascade/common.py b/comfy/ldm/cascade/common.py
index 42ef98c7a..c42a486b7 100644
--- a/comfy/ldm/cascade/common.py
+++ b/comfy/ldm/cascade/common.py
@@ -18,12 +18,13 @@
import torch
import torch.nn as nn
-from comfy.ldm.modules.attention import optimized_attention
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention
import comfy.ops
class OptimizedAttention(nn.Module):
def __init__(self, c, nhead, dropout=0.0, dtype=None, device=None, operations=None):
super().__init__()
+ self.comfy_attention = ComfyAttention()
self.heads = nhead
self.to_q = operations.Linear(c, c, bias=True, dtype=dtype, device=device)
@@ -37,7 +38,8 @@ class OptimizedAttention(nn.Module):
k = self.to_k(k)
v = self.to_v(v)
- out = optimized_attention(q, k, v, self.heads, transformer_options=transformer_options)
+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)
+ out = optimized_attention(q, k, v, self.heads, preferred_attention=self.comfy_attention, transformer_options=transformer_options)
return self.out_proj(out)
diff --git a/comfy/ldm/cogvideo/model.py b/comfy/ldm/cogvideo/model.py
index fb475ed53..a9e09b257 100644
--- a/comfy/ldm/cogvideo/model.py
+++ b/comfy/ldm/cogvideo/model.py
@@ -7,7 +7,7 @@ import torch
import torch.nn as nn
import torch.nn.functional as F
-from comfy.ldm.modules.attention import optimized_attention
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention
import comfy.patcher_extension
import comfy.ldm.common_dit
@@ -241,6 +241,7 @@ class CogVideoXBlock(nn.Module):
eps=1e-5, ff_inner_dim=None, ff_bias=True,
device=None, dtype=None, operations=None):
super().__init__()
+ self.comfy_attention = ComfyAttention()
self.dim = dim
self.num_heads = num_heads
self.head_dim = head_dim
@@ -290,12 +291,15 @@ class CogVideoXBlock(nn.Module):
k_img = apply_rotary_emb(k_img, image_rotary_emb)
q = torch.cat([q[:, :text_seq_length], q_img.transpose(1, 2)], dim=1)
k = torch.cat([k[:, :text_seq_length], k_img.transpose(1, 2)], dim=1)
+ del q_img, k_img
+ q, k, v = AttentionTensorContainer(q.reshape(b, s, n * d)), AttentionTensorContainer(k.reshape(b, s, n * d)), AttentionTensorContainer(v)
attn_out = optimized_attention(
- q.reshape(b, s, n * d),
- k.reshape(b, s, n * d),
+ q,
+ k,
v,
heads=self.num_heads,
+ preferred_attention=self.comfy_attention,
transformer_options=transformer_options,
)
diff --git a/comfy/ldm/cosmos/blocks.py b/comfy/ldm/cosmos/blocks.py
index afb43d469..4faae2541 100644
--- a/comfy/ldm/cosmos/blocks.py
+++ b/comfy/ldm/cosmos/blocks.py
@@ -23,7 +23,9 @@ from einops import rearrange, repeat
from einops.layers.torch import Rearrange
from torch import nn
-from comfy.ldm.modules.attention import optimized_attention
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention
+import comfy.model_management
+import comfy.quant_ops
def get_normalization(name: str, channels: int, weight_args={}, operations=None):
@@ -89,6 +91,7 @@ class Attention(nn.Module):
operations=None,
) -> None:
super().__init__()
+ self.comfy_attention = ComfyAttention()
self.is_selfattn = context_dim is None # self attention
@@ -157,17 +160,20 @@ class Attention(nn.Module):
k = self.to_k[1](k)
v = self.to_v[1](v)
if self.is_selfattn and rope_emb is not None: # only apply to self-attention!
- # apply_rotary_pos_emb inlined
- q_shape = q.shape
- q = q.reshape(*q.shape[:-1], 2, -1).movedim(-2, -1).unsqueeze(-2)
- q = rope_emb[..., 0] * q[..., 0] + rope_emb[..., 1] * q[..., 1]
- q = q.movedim(-1, -2).reshape(*q_shape).to(x.dtype)
-
- # apply_rotary_pos_emb inlined
- k_shape = k.shape
- k = k.reshape(*k.shape[:-1], 2, -1).movedim(-2, -1).unsqueeze(-2)
- k = rope_emb[..., 0] * k[..., 0] + rope_emb[..., 1] * k[..., 1]
- k = k.movedim(-1, -2).reshape(*k_shape).to(x.dtype)
+ if not comfy.model_management.in_training:
+ q, k = comfy.quant_ops.ck.apply_rope_split_half(q, k, rope_emb)
+ else:
+ # apply_rotary_pos_emb inlined
+ q_shape = q.shape
+ q = q.reshape(*q.shape[:-1], 2, -1).movedim(-2, -1).unsqueeze(-2)
+ q = rope_emb[..., 0] * q[..., 0] + rope_emb[..., 1] * q[..., 1]
+ q = q.movedim(-1, -2).reshape(*q_shape).to(x.dtype)
+
+ # apply_rotary_pos_emb inlined
+ k_shape = k.shape
+ k = k.reshape(*k.shape[:-1], 2, -1).movedim(-2, -1).unsqueeze(-2)
+ k = rope_emb[..., 0] * k[..., 0] + rope_emb[..., 1] * k[..., 1]
+ k = k.movedim(-1, -2).reshape(*k_shape).to(x.dtype)
return q, k, v
def forward(
@@ -185,7 +191,8 @@ class Attention(nn.Module):
context (Optional[Tensor]): The key tensor of shape [B, Mk, K] or use x as context [self attention] if None
"""
q, k, v = self.cal_qkv(x, context, mask, rope_emb=rope_emb, **kwargs)
- out = optimized_attention(q, k, v, self.heads, skip_reshape=True, mask=mask, skip_output_reshape=True, transformer_options=transformer_options)
+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)
+ out = optimized_attention(q, k, v, self.heads, skip_reshape=True, mask=mask, skip_output_reshape=True, preferred_attention=self.comfy_attention, transformer_options=transformer_options)
del q, k, v
out = rearrange(out, " b n s c -> s b (n c)")
return self.to_out(out)
diff --git a/comfy/ldm/cosmos/predict2.py b/comfy/ldm/cosmos/predict2.py
index 0f4648202..8a5c29b7d 100644
--- a/comfy/ldm/cosmos/predict2.py
+++ b/comfy/ldm/cosmos/predict2.py
@@ -12,7 +12,7 @@ from .position_embedding import VideoRopePosition3DEmb, LearnablePosEmbAxis
from torchvision import transforms
import comfy.patcher_extension
-from comfy.ldm.modules.attention import optimized_attention
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention
import comfy.ldm.common_dit
import comfy.ops
import comfy.quant_ops
@@ -38,7 +38,7 @@ class GPT2FeedForward(nn.Module):
return x
-def torch_attention_op(q_B_S_H_D: torch.Tensor, k_B_S_H_D: torch.Tensor, v_B_S_H_D: torch.Tensor, transformer_options: Optional[dict] = {}) -> torch.Tensor:
+def torch_attention_op(q_B_S_H_D: torch.Tensor, k_B_S_H_D: torch.Tensor, v_B_S_H_D: torch.Tensor, transformer_options: Optional[dict] = {}, preferred_attention=None) -> torch.Tensor:
"""Computes multi-head attention using PyTorch's native implementation.
This function provides a PyTorch backend alternative to Transformer Engine's attention operation.
@@ -60,12 +60,16 @@ def torch_attention_op(q_B_S_H_D: torch.Tensor, k_B_S_H_D: torch.Tensor, v_B_S_H
Returns:
Attention output tensor with shape (batch, seq_len, n_heads * head_dim)
"""
+ if isinstance(q_B_S_H_D, AttentionTensorContainer):
+ q_B_S_H_D, k_B_S_H_D, v_B_S_H_D = q_B_S_H_D.take(), k_B_S_H_D.take(), v_B_S_H_D.take()
in_q_shape = q_B_S_H_D.shape
in_k_shape = k_B_S_H_D.shape
q_B_H_S_D = rearrange(q_B_S_H_D, "b ... h k -> b h ... k").view(in_q_shape[0], in_q_shape[-2], -1, in_q_shape[-1])
k_B_H_S_D = rearrange(k_B_S_H_D, "b ... h v -> b h ... v").view(in_k_shape[0], in_k_shape[-2], -1, in_k_shape[-1])
v_B_H_S_D = rearrange(v_B_S_H_D, "b ... h v -> b h ... v").view(in_k_shape[0], in_k_shape[-2], -1, in_k_shape[-1])
- return optimized_attention(q_B_H_S_D, k_B_H_S_D, v_B_H_S_D, in_q_shape[-2], skip_reshape=True, transformer_options=transformer_options)
+ q_B_H_S_D, k_B_H_S_D, v_B_H_S_D = AttentionTensorContainer(q_B_H_S_D), AttentionTensorContainer(k_B_H_S_D), AttentionTensorContainer(v_B_H_S_D)
+ del q_B_S_H_D, k_B_S_H_D, v_B_S_H_D
+ return optimized_attention(q_B_H_S_D, k_B_H_S_D, v_B_H_S_D, in_q_shape[-2], skip_reshape=True, preferred_attention=preferred_attention, transformer_options=transformer_options)
class Attention(nn.Module):
@@ -112,6 +116,7 @@ class Attention(nn.Module):
operations=None,
) -> None:
super().__init__()
+ self.comfy_attention = ComfyAttention()
logging.debug(
f"Setting up {self.__class__.__name__}. Query dim is {query_dim}, context_dim is {context_dim} and using "
f"{n_heads} heads with a dimension of {head_dim}."
@@ -197,7 +202,7 @@ class Attention(nn.Module):
return q, k, v
def compute_attention(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, transformer_options: Optional[dict] = {}) -> torch.Tensor:
- result = self.attn_op(q, k, v, transformer_options=transformer_options) # [B, S, H, D]
+ result = self.attn_op(q, k, v, transformer_options=transformer_options, preferred_attention=self.comfy_attention) # [B, S, H, D]
return self.output_dropout(self.output_proj(result))
def forward(
@@ -213,6 +218,7 @@ class Attention(nn.Module):
context (Optional[Tensor]): The key tensor of shape [B, Mk, K] or use x as context [self attention] if None
"""
q, k, v = self.compute_qkv(x, context, rope_emb=rope_emb, transformer_options=transformer_options)
+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)
return self.compute_attention(q, k, v, transformer_options=transformer_options)
diff --git a/comfy/ldm/genmo/joint_model/asymm_models_joint.py b/comfy/ldm/genmo/joint_model/asymm_models_joint.py
index 5c1bb4d42..255944894 100644
--- a/comfy/ldm/genmo/joint_model/asymm_models_joint.py
+++ b/comfy/ldm/genmo/joint_model/asymm_models_joint.py
@@ -8,7 +8,7 @@ import torch.nn as nn
import torch.nn.functional as F
from einops import rearrange
# from flash_attn import flash_attn_varlen_qkvpacked_func
-from comfy.ldm.modules.attention import optimized_attention
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention
from .layers import (
FeedForward,
@@ -68,6 +68,7 @@ class AsymmetricAttention(nn.Module):
operations=None,
):
super().__init__()
+ self.comfy_attention = ComfyAttention()
self.dim_x = dim_x
self.dim_y = dim_y
self.num_heads = num_heads
@@ -142,13 +143,16 @@ class AsymmetricAttention(nn.Module):
k = torch.cat([k_x, k_y[:, :crop_y]], dim=1).transpose(1, 2)
v = torch.cat([v_x, v_y[:, :crop_y]], dim=1).transpose(1, 2)
+ seq_x, seq_y = q_x.shape[1], q_y.shape[1]
+ del q_x, k_x, v_x, q_y, k_y, v_y
+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)
xy = optimized_attention(q,
k,
- v, self.num_heads, skip_reshape=True, transformer_options=transformer_options)
+ v, self.num_heads, skip_reshape=True, preferred_attention=self.comfy_attention, transformer_options=transformer_options)
- x, y = torch.tensor_split(xy, (q_x.shape[1],), dim=1)
+ x, y = torch.tensor_split(xy, (seq_x,), dim=1)
x = self.proj_x(x)
- o = torch.zeros(y.shape[0], q_y.shape[1], y.shape[-1], device=y.device, dtype=y.dtype)
+ o = torch.zeros(y.shape[0], seq_y, y.shape[-1], device=y.device, dtype=y.dtype)
o[:, :y.shape[1]] = y
y = self.proj_y(o)
diff --git a/comfy/ldm/genmo/joint_model/utils.py b/comfy/ldm/genmo/joint_model/utils.py
index 1b399d5d2..a720d758c 100644
--- a/comfy/ldm/genmo/joint_model/utils.py
+++ b/comfy/ldm/genmo/joint_model/utils.py
@@ -4,6 +4,7 @@
from typing import Optional
import torch
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention
import torch.nn as nn
import torch.nn.functional as F
@@ -51,6 +52,7 @@ class AttentionPool(nn.Module):
output_dim (int): Dimensionality of output tokens. Defaults to embed_dim.
"""
super().__init__()
+ self.comfy_attention = ComfyAttention()
self.num_heads = num_heads
self.to_kv = operations.Linear(embed_dim, 2 * embed_dim, device=device, dtype=dtype)
self.to_q = operations.Linear(embed_dim, embed_dim, device=device, dtype=dtype)
@@ -92,9 +94,9 @@ class AttentionPool(nn.Module):
q = q.unsqueeze(2) # (B, H, 1, head_dim)
# Compute attention.
- x = F.scaled_dot_product_attention(
- q, k, v, attn_mask=attn_mask, dropout_p=0.0
- ) # (B, H, 1, head_dim)
+ del kv
+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)
+ x = optimized_attention(q, k, v, self.num_heads, mask=attn_mask, skip_reshape=True, skip_output_reshape=True, preferred_attention=self.comfy_attention)
# Concatenate heads and run output.
x = x.squeeze(2).flatten(1, 2) # (B, D = H * head_dim)
diff --git a/comfy/ldm/hunyuan_video/model.py b/comfy/ldm/hunyuan_video/model.py
index b94cdfa87..12aa98606 100644
--- a/comfy/ldm/hunyuan_video/model.py
+++ b/comfy/ldm/hunyuan_video/model.py
@@ -4,7 +4,7 @@ import torch
import comfy.patcher_extension
import comfy.ldm.flux.layers
import comfy.ldm.modules.diffusionmodules.mmdit
-from comfy.ldm.modules.attention import optimized_attention
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention
from dataclasses import dataclass
from einops import repeat
@@ -63,6 +63,7 @@ class TokenRefinerBlock(nn.Module):
operations=None
):
super().__init__()
+ self.comfy_attention = ComfyAttention()
self.heads = heads
mlp_hidden_dim = hidden_size * 4
@@ -88,7 +89,9 @@ class TokenRefinerBlock(nn.Module):
norm_x = self.norm1(x)
qkv = self.self_attn.qkv(norm_x)
q, k, v = qkv.reshape(qkv.shape[0], qkv.shape[1], 3, self.heads, -1).permute(2, 0, 3, 1, 4)
- attn = optimized_attention(q, k, v, self.heads, mask=mask, skip_reshape=True, transformer_options=transformer_options)
+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)
+ del qkv
+ attn = optimized_attention(q, k, v, self.heads, mask=mask, skip_reshape=True, preferred_attention=self.comfy_attention, transformer_options=transformer_options)
x = x + self.self_attn.proj(attn) * mod1.unsqueeze(1)
x = x + self.mlp(self.norm2(x)) * mod2.unsqueeze(1)
diff --git a/comfy/ldm/ideogram4/model.py b/comfy/ldm/ideogram4/model.py
index 12e1a14fb..f9232baf3 100644
--- a/comfy/ldm/ideogram4/model.py
+++ b/comfy/ldm/ideogram4/model.py
@@ -17,7 +17,7 @@ import comfy.ops
import comfy.patcher_extension
import comfy.quant_ops
from comfy.ldm.lumina.model import FeedForward
-from comfy.ldm.modules.attention import optimized_attention_masked
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention_masked
from comfy.text_encoders.llama import precompute_freqs_cis
# Per-token role indicators
@@ -47,6 +47,7 @@ def _apply_rope_split_half1(x, freqs_cis):
class Ideogram4Attention(nn.Module):
def __init__(self, hidden_size, num_heads, eps=1e-5, dtype=None, device=None, operations=None):
super().__init__()
+ self.comfy_attention = ComfyAttention()
self.num_heads = num_heads
self.head_dim = hidden_size // num_heads
self.hidden_size = hidden_size
@@ -74,11 +75,12 @@ class Ideogram4Attention(nn.Module):
comfy.ops.uncast_bias_weight(self.norm_k, k_scale, None, k_offload_stream)
# (B, heads, L, head_dim)
- q = q.transpose(1, 2)
- k = k.transpose(1, 2)
- v = v.transpose(1, 2)
+ q = AttentionTensorContainer(q.transpose(1, 2))
+ k = AttentionTensorContainer(k.transpose(1, 2))
+ v = AttentionTensorContainer(v.transpose(1, 2))
+ del qkv
- out = optimized_attention_masked(q, k, v, self.num_heads, attn_mask, skip_reshape=True, transformer_options=transformer_options)
+ out = optimized_attention_masked(q, k, v, self.num_heads, attn_mask, skip_reshape=True, preferred_attention=self.comfy_attention, transformer_options=transformer_options)
return self.o(out)
diff --git a/comfy/ldm/joyimage/model.py b/comfy/ldm/joyimage/model.py
index 9d6951e54..148b95d9a 100644
--- a/comfy/ldm/joyimage/model.py
+++ b/comfy/ldm/joyimage/model.py
@@ -10,7 +10,7 @@ import comfy.ldm.common_dit
import comfy.ops
import comfy.patcher_extension
from comfy.ldm.lightricks.model import GELU_approx, PixArtAlphaTextProjection, TimestepEmbedding, Timesteps
-from comfy.ldm.modules.attention import optimized_attention
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention
class JoyImageModulate(nn.Module):
@@ -63,6 +63,7 @@ class JoyImageAttention(nn.Module):
):
super().__init__()
self.num_attention_heads = num_attention_heads
+ self.comfy_attention = ComfyAttention()
inner_dim = num_attention_heads * attention_head_dim
self.img_attn_qkv = operations.Linear(dim, inner_dim * 3, bias=True, dtype=dtype, device=device)
@@ -113,12 +114,13 @@ class JoyImageAttention(nn.Module):
joint_q = torch.cat([img_q, txt_q], dim=1)
joint_k = torch.cat([img_k, txt_k], dim=1)
joint_v = torch.cat([img_v, txt_v], dim=1)
+ del img_q, img_k, img_v, txt_q, txt_k, txt_v
- joint_q = joint_q.flatten(2, 3)
- joint_k = joint_k.flatten(2, 3)
- joint_v = joint_v.flatten(2, 3)
+ joint_q = AttentionTensorContainer(joint_q.flatten(2, 3))
+ joint_k = AttentionTensorContainer(joint_k.flatten(2, 3))
+ joint_v = AttentionTensorContainer(joint_v.flatten(2, 3))
- joint_out = optimized_attention(joint_q, joint_k, joint_v, heads=heads, transformer_options=transformer_options)
+ joint_out = optimized_attention(joint_q, joint_k, joint_v, heads=heads, preferred_attention=self.comfy_attention, transformer_options=transformer_options)
seq_img = img.shape[1]
img_out = joint_out[:, :seq_img, :]
diff --git a/comfy/ldm/kandinsky5/model.py b/comfy/ldm/kandinsky5/model.py
index 1509de2f8..32908b5f6 100644
--- a/comfy/ldm/kandinsky5/model.py
+++ b/comfy/ldm/kandinsky5/model.py
@@ -3,16 +3,20 @@ from torch import nn
import math
import comfy.ldm.common_dit
-from comfy.ldm.modules.attention import optimized_attention
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention
from comfy.ldm.flux.math import apply_rope1
from comfy.ldm.flux.layers import EmbedND
-def attention(q, k, v, heads, transformer_options={}):
+def attention(q, k, v, heads, transformer_options={}, preferred_attention=None):
+ if isinstance(q, AttentionTensorContainer):
+ q, k, v = q.take(), k.take(), v.take()
+ q = AttentionTensorContainer(q.transpose(1, 2))
+ k = AttentionTensorContainer(k.transpose(1, 2))
+ v = AttentionTensorContainer(v.transpose(1, 2))
return optimized_attention(
- q.transpose(1, 2),
- k.transpose(1, 2),
- v.transpose(1, 2),
+ q, k, v,
heads=heads,
+ preferred_attention=preferred_attention,
skip_reshape=True,
transformer_options=transformer_options
)
@@ -98,6 +102,7 @@ class Modulation(nn.Module):
class SelfAttention(nn.Module):
def __init__(self, num_channels, head_dim, operation_settings=None):
super().__init__()
+ self.comfy_attention = ComfyAttention()
assert num_channels % head_dim == 0
self.num_heads = num_channels // head_dim
self.head_dim = head_dim
@@ -120,7 +125,8 @@ class SelfAttention(nn.Module):
q = self._compute_qk(x, freqs, self.to_query, self.query_norm)
k = self._compute_qk(x, freqs, self.to_key, self.key_norm)
v = self.to_value(x).view(*x.shape[:-1], self.num_heads, -1)
- out = attention(q, k, v, self.num_heads, transformer_options=transformer_options)
+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)
+ out = attention(q, k, v, self.num_heads, preferred_attention=self.comfy_attention, transformer_options=transformer_options)
return self.out_layer(out)
def _forward_chunked(self, x, freqs, transformer_options={}):
@@ -135,7 +141,8 @@ class SelfAttention(nn.Module):
q = process_chunks(self.to_query, self.query_norm)
k = process_chunks(self.to_key, self.key_norm)
v = self.to_value(x).view(*x.shape[:-1], self.num_heads, -1)
- out = attention(q, k, v, self.num_heads, transformer_options=transformer_options)
+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)
+ out = attention(q, k, v, self.num_heads, preferred_attention=self.comfy_attention, transformer_options=transformer_options)
return self.out_layer(out)
def forward(self, x, freqs, transformer_options={}):
@@ -154,7 +161,8 @@ class CrossAttention(SelfAttention):
def forward(self, x, context, transformer_options={}):
q, k, v = self.get_qkv(x, context)
- out = attention(self.query_norm(q), self.key_norm(k), v, self.num_heads, transformer_options=transformer_options)
+ q, k, v = AttentionTensorContainer(self.query_norm(q)), AttentionTensorContainer(self.key_norm(k)), AttentionTensorContainer(v)
+ out = attention(q, k, v, self.num_heads, preferred_attention=self.comfy_attention, transformer_options=transformer_options)
return self.out_layer(out)
diff --git a/comfy/ldm/krea2/model.py b/comfy/ldm/krea2/model.py
index 8001812d7..a904e05e0 100644
--- a/comfy/ldm/krea2/model.py
+++ b/comfy/ldm/krea2/model.py
@@ -18,7 +18,7 @@ import comfy.ldm.common_dit
import comfy.utils
from comfy.ldm.flux.layers import EmbedND, timestep_embedding
from comfy.ldm.flux.math import apply_rope
-from comfy.ldm.modules.attention import optimized_attention_masked
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention_masked
class RMSNorm(nn.Module):
@@ -63,6 +63,7 @@ class Attention(nn.Module):
def __init__(self, dim: int, heads: int, kvheads: Optional[int] = None, bias: bool = False,
device=None, dtype=None, operations=None):
super().__init__()
+ self.comfy_attention = ComfyAttention()
self.heads = heads
self.kvheads = kvheads if kvheads is not None else heads
self.headdim = dim // self.heads
@@ -94,8 +95,9 @@ class Attention(nn.Module):
rep = self.heads // self.kvheads
k = k.repeat_interleave(rep, dim=1)
v = v.repeat_interleave(rep, dim=1)
+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)
out = optimized_attention_masked(q, k, v, self.heads, mask=mask, skip_reshape=True,
- transformer_options=transformer_options)
+ preferred_attention=self.comfy_attention, transformer_options=transformer_options)
if "block_index" in transformer_options and "attn1_output_patch" in transformer_patches:
for p in transformer_patches["attn1_output_patch"]:
diff --git a/comfy/ldm/modules/attention.py b/comfy/ldm/modules/attention.py
index 7e4f94351..c54415368 100644
--- a/comfy/ldm/modules/attention.py
+++ b/comfy/ldm/modules/attention.py
@@ -956,6 +956,7 @@ def optimized_attention_for_device(device, mask=False, small_input=False):
class CrossAttention(nn.Module):
def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0., attn_precision=None, dtype=None, device=None, operations=ops):
super().__init__()
+ self.comfy_attention = ComfyAttention()
inner_dim = dim_head * heads
context_dim = default(context_dim, query_dim)
self.attn_precision = attn_precision
@@ -979,10 +980,11 @@ class CrossAttention(nn.Module):
else:
v = self.to_v(context)
+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)
if mask is None:
- out = optimized_attention(q, k, v, self.heads, attn_precision=self.attn_precision, transformer_options=transformer_options)
+ out = optimized_attention(q, k, v, self.heads, attn_precision=self.attn_precision, preferred_attention=self.comfy_attention, transformer_options=transformer_options)
else:
- out = optimized_attention_masked(q, k, v, self.heads, mask, attn_precision=self.attn_precision, transformer_options=transformer_options)
+ out = optimized_attention_masked(q, k, v, self.heads, mask, attn_precision=self.attn_precision, preferred_attention=self.comfy_attention, transformer_options=transformer_options)
return self.to_out(out)
diff --git a/comfy/ldm/sensenova/model.py b/comfy/ldm/sensenova/model.py
index 9dffff301..45a43b8b0 100644
--- a/comfy/ldm/sensenova/model.py
+++ b/comfy/ldm/sensenova/model.py
@@ -6,7 +6,7 @@ import comfy.patcher_extension
import comfy.utils
from comfy.ldm.common_dit import pad_to_patch_size
from comfy.ldm.flux.math import apply_rope1
-from comfy.ldm.modules.attention import optimized_attention
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention
from comfy.ldm.modules.diffusionmodules.mmdit import TimestepEmbedder
from .sampling import resolution_noise_scale
@@ -190,6 +190,7 @@ class MLP(nn.Module):
class Attention(nn.Module):
def __init__(self, device=None, dtype=None, operations=None):
super().__init__()
+ self.comfy_attention = ComfyAttention()
self.q_proj = operations.Linear(
HIDDEN_SIZE, NUM_HEADS * HEAD_DIM, bias=False, device=device, dtype=dtype
)
@@ -288,15 +289,16 @@ class Attention(nn.Module):
self, hidden_states, rope, attention_mask, transformer_options
):
query, key, value = self._project(hidden_states, rope, False)
+ query = AttentionTensorContainer(query)
output = optimized_attention(
query,
- key,
- value,
+ AttentionTensorContainer(key),
+ AttentionTensorContainer(value),
NUM_HEADS,
mask=attention_mask,
skip_reshape=True,
transformer_options=transformer_options,
- enable_gqa=True,
+ enable_gqa=True, preferred_attention=self.comfy_attention,
)
return self.o_proj(output), key, value
@@ -306,6 +308,7 @@ class Attention(nn.Module):
query, key, value = self._project(hidden_states, rope, True)
key = torch.cat((prefix_key, key), dim=2)
value = torch.cat((prefix_value, value), dim=2)
+ query, key, value = AttentionTensorContainer(query), AttentionTensorContainer(key), AttentionTensorContainer(value)
output = optimized_attention(
query,
key,
@@ -314,7 +317,7 @@ class Attention(nn.Module):
mask=None,
skip_reshape=True,
transformer_options=transformer_options,
- enable_gqa=True,
+ enable_gqa=True, preferred_attention=self.comfy_attention,
)
return self.o_proj_mot_gen(output)
diff --git a/comfy/ldm/trellis2/model.py b/comfy/ldm/trellis2/model.py
index fda7dfd37..352c9c91c 100644
--- a/comfy/ldm/trellis2/model.py
+++ b/comfy/ldm/trellis2/model.py
@@ -5,16 +5,19 @@ import comfy.model_management
import comfy.ops
from comfy.ldm.trellis2.vae import SparseTensor, SparseLinear, sparse_cat, VarLenTensor
from typing import Optional, Tuple, Literal, Union, List
-from comfy.ldm.modules.attention import optimized_attention
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention
from comfy.ldm.genmo.joint_model.layers import TimestepEmbedder
from comfy.ldm.flux.math import apply_rope, apply_rope1
def dense_attention(q, k, v, **kwargs):
+ if isinstance(q, AttentionTensorContainer):
+ q, k, v = q.take(), k.take(), v.take()
heads = q.shape[2]
q = q.permute(0, 2, 1, 3)
k = k.permute(0, 2, 1, 3)
v = v.permute(0, 2, 1, 3)
+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)
out = optimized_attention(q, k, v, heads, skip_output_reshape=True, skip_reshape=True, **kwargs)
return out.permute(0, 2, 1, 3)
@@ -35,6 +38,8 @@ def _to_padded(t):
def sparse_attention(q, k, v, **kwargs):
+ if isinstance(q, AttentionTensorContainer):
+ q, k, v = q.take(), k.take(), v.take()
q_padded, q_valid = _to_padded(q)
k_padded, k_valid = _to_padded(k)
v_padded, _ = _to_padded(v)
@@ -43,6 +48,8 @@ def sparse_attention(q, k, v, **kwargs):
dtype=q_padded.dtype, device=q_padded.device)
mask.masked_fill_(~k_valid[:, None, None, :], -torch.finfo(q_padded.dtype).max)
kwargs["mask"] = mask
+ q_padded, k_padded, v_padded = AttentionTensorContainer(q_padded), AttentionTensorContainer(k_padded), AttentionTensorContainer(v_padded)
+ del k, v
out = dense_attention(q_padded, k_padded, v_padded, **kwargs)
if isinstance(q, VarLenTensor):
if q_valid is not None:
@@ -146,6 +153,7 @@ class SparseMultiHeadAttention(nn.Module):
device=None, dtype=None, operations=None
):
super().__init__()
+ self.comfy_attention = ComfyAttention()
self.channels = channels
self.head_dim = channels // num_heads
@@ -199,18 +207,22 @@ class SparseMultiHeadAttention(nn.Module):
if self.qk_rms_norm:
q = self.q_rms_norm(q)
k = self.k_rms_norm(k)
+ del qkv
q, k = self.rope(q, k)
- h = sparse_attention(q, k, v, transformer_options=transformer_options)
+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)
+ h = sparse_attention(q, k, v, transformer_options=transformer_options, preferred_attention=self.comfy_attention)
else:
q = self._linear(self.to_q, x)
q = self._reshape_chs(q, (self.num_heads, -1))
kv = self._linear(self.to_kv, context)
kv = self._fused_pre(kv, num_fused=2)
k, v = kv.unbind(dim=-3)
+ del kv
if self.qk_rms_norm:
q = self.q_rms_norm(q)
k = self.k_rms_norm(k)
- h = sparse_attention(q, k, v, transformer_options=transformer_options)
+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)
+ h = sparse_attention(q, k, v, transformer_options=transformer_options, preferred_attention=self.comfy_attention)
h = self._reshape_chs(h, (-1,))
h = self._linear(self.to_out, h)
return h
@@ -471,6 +483,7 @@ class MultiHeadAttention(nn.Module):
device=None, dtype=None, operations=None
):
super().__init__()
+ self.comfy_attention = ComfyAttention()
self.channels = channels
self.head_dim = channels // num_heads
@@ -505,8 +518,10 @@ class MultiHeadAttention(nn.Module):
# phases is [L, head_dim/2, 2, 2]; broadcast to [1, L, 1, ...]
# to align with q/k of shape [B, L, H, head_dim].
f_cis = comfy.model_management.cast_to(phases, device=q.device).unsqueeze(0).unsqueeze(2)
+ del qkv
q, k = apply_rope(q, k, f_cis)
- h = dense_attention(q, k, v, transformer_options=transformer_options)
+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)
+ h = dense_attention(q, k, v, transformer_options=transformer_options, preferred_attention=self.comfy_attention)
else:
Lkv = context.shape[1]
q = self.to_q(x)
@@ -514,10 +529,12 @@ class MultiHeadAttention(nn.Module):
q = q.reshape(B, L, self.num_heads, -1)
kv = kv.reshape(B, Lkv, 2, self.num_heads, -1)
k, v = kv.unbind(dim=2)
+ del kv
if self.qk_rms_norm:
q = self.q_rms_norm(q)
k = self.k_rms_norm(k)
- h = dense_attention(q, k, v, transformer_options=transformer_options)
+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)
+ h = dense_attention(q, k, v, transformer_options=transformer_options, preferred_attention=self.comfy_attention)
h = h.reshape(B, L, -1)
h = self.to_out(h)
return h
diff --git a/comfy/ldm/triposplat/model.py b/comfy/ldm/triposplat/model.py
index d8a531772..9f4cdb830 100644
--- a/comfy/ldm/triposplat/model.py
+++ b/comfy/ldm/triposplat/model.py
@@ -9,7 +9,7 @@ import torch.nn.functional as F
import comfy.model_management
import comfy.patcher_extension
import comfy.rmsnorm
-from comfy.ldm.modules.attention import optimized_attention
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention
from comfy.ldm.flux.math import apply_rope
@@ -100,9 +100,13 @@ class PcdAbsolutePositionEmbedder(nn.Module):
return out.to(orig_dtype)
-def attention(q, k, v, transformer_options=None):
+def attention(q, k, v, transformer_options=None, preferred_attention=None):
+ if isinstance(q, AttentionTensorContainer):
+ q, k, v = q.take(), k.take(), v.take()
+ heads = q.shape[2]
+ q, k, v = AttentionTensorContainer(q.transpose(1, 2)), AttentionTensorContainer(k.transpose(1, 2)), AttentionTensorContainer(v.transpose(1, 2))
# q, k, v: (B, L, heads, dim) -> (B, L, heads, dim). Shared optimized_attention call convention.
- out = optimized_attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), heads=q.shape[2],
+ out = optimized_attention(q, k, v, heads=heads, preferred_attention=preferred_attention,
skip_reshape=True, skip_output_reshape=True, low_precision_attention=False,
transformer_options=transformer_options)
return out.transpose(1, 2)
@@ -127,6 +131,7 @@ class RopeMultiHeadAttention(nn.Module):
def __init__(self, channels, num_heads, qkv_bias=True, qk_rms_norm=False, use_rope=False,
dtype=None, device=None, operations=None):
super().__init__()
+ self.comfy_attention = ComfyAttention()
self.channels = channels
self.num_heads = num_heads
self.head_dim = channels // num_heads
@@ -147,7 +152,9 @@ class RopeMultiHeadAttention(nn.Module):
if self.qk_rms_norm:
q = self.q_norm(q)
k = self.k_norm(k)
- h = attention(q, k, v, transformer_options) # (B, L, heads, dim)
+ del qkv
+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)
+ h = attention(q, k, v, transformer_options, preferred_attention=self.comfy_attention) # (B, L, heads, dim)
return self.out(h.reshape(B, L, C))
diff --git a/comfy/ldm/wan/model.py b/comfy/ldm/wan/model.py
index 1ac306368..0925ac70c 100644
--- a/comfy/ldm/wan/model.py
+++ b/comfy/ldm/wan/model.py
@@ -6,13 +6,14 @@ import torch
import torch.nn as nn
from einops import rearrange
-from comfy.ldm.modules.attention import AttentionTensorContainer, optimized_attention
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention
from comfy.ldm.flux.layers import EmbedND
from comfy.ldm.flux.math import apply_rope1, rope
import comfy.ldm.common_dit
import comfy.model_management
import comfy.ops
import comfy.patcher_extension
+import comfy.quant_ops
def sinusoidal_embedding_1d(dim, position):
@@ -40,6 +41,7 @@ class WanSelfAttention(nn.Module):
operation_settings={}):
assert dim % num_heads == 0
super().__init__()
+ self.comfy_attention = ComfyAttention()
self.dim = dim
self.num_heads = num_heads
self.head_dim = dim // num_heads
@@ -90,6 +92,7 @@ class WanSelfAttention(nn.Module):
x = optimized_attention(
q, k, v,
heads=self.num_heads,
+ preferred_attention=self.comfy_attention,
transformer_options=transformer_options,
)
@@ -115,7 +118,7 @@ class WanT2VCrossAttention(WanSelfAttention):
v = AttentionTensorContainer(self.v(context))
# compute attention
- x = optimized_attention(q, k, v, heads=self.num_heads, transformer_options=transformer_options)
+ x = optimized_attention(q, k, v, heads=self.num_heads, preferred_attention=self.comfy_attention, transformer_options=transformer_options)
x = self.o(x)
return x
@@ -147,16 +150,16 @@ class WanI2VCrossAttention(WanSelfAttention):
# compute query, key, value
q = self.norm_q(self.q(x))
- k_img = self.norm_k_img(self.k_img(context_img))
- v_img = self.v_img(context_img)
+ k_img = AttentionTensorContainer(self.norm_k_img(self.k_img(context_img)))
+ v_img = AttentionTensorContainer(self.v_img(context_img))
# Sageattn can cause Nans here, don't allow it as there is no speed difference anyway as img attention is tiny.
- img_x = optimized_attention(q, k_img, v_img, heads=self.num_heads, transformer_options=transformer_options, low_precision_attention=False)
+ img_x = optimized_attention(AttentionTensorContainer(q), k_img, v_img, heads=self.num_heads, preferred_attention=self.comfy_attention, transformer_options=transformer_options, low_precision_attention=False)
del k_img, v_img
# compute attention
q = AttentionTensorContainer(q)
k = AttentionTensorContainer(self.norm_k(self.k(context)))
v = AttentionTensorContainer(self.v(context))
- x = optimized_attention(q, k, v, heads=self.num_heads, transformer_options=transformer_options)
+ x = optimized_attention(q, k, v, heads=self.num_heads, preferred_attention=self.comfy_attention, transformer_options=transformer_options)
# output
x = x + img_x
@@ -182,6 +185,13 @@ def repeat_e(e, x):
return torch.repeat_interleave(e, repeats + 1, dim=1)[:, :x.size(1)]
+def modulate(x, norm, shift, scale):
+ shift, scale = repeat_e(shift, x), repeat_e(scale, x)
+ if comfy.model_management.in_training or not x.is_cuda or x.numel() < 16 * 1024 * 1024:
+ return torch.addcmul(shift, norm(x), 1 + scale)
+ return comfy.quant_ops.ck.adaln(x, scale, shift, norm.eps)
+
+
class WanFeedForward(nn.Sequential):
"""[Linear, GELU(tanh), Linear], with the GELU folded into the down-projection."""
@@ -257,7 +267,7 @@ class WanAttentionBlock(nn.Module):
# self-attention
x = x.contiguous() # otherwise implicit in LayerNorm
y = self.self_attn(
- torch.addcmul(repeat_e(e[0], x), self.norm1(x), 1 + repeat_e(e[1], x)),
+ modulate(x, self.norm1, e[0], e[1]),
freqs, transformer_options=transformer_options)
x = torch.addcmul(x, y, repeat_e(e[2], x))
@@ -270,7 +280,7 @@ class WanAttentionBlock(nn.Module):
for p in patches["attn2_patch"]:
x = p({"x": x, "transformer_options": transformer_options})
- y = self.ffn(torch.addcmul(repeat_e(e[3], x), self.norm2(x), 1 + repeat_e(e[4], x)))
+ y = self.ffn(modulate(x, self.norm2, e[3], e[4]))
x = torch.addcmul(x, y, repeat_e(e[5], x))
return x
@@ -386,7 +396,7 @@ class Head(nn.Module):
else:
e = (comfy.model_management.cast_to(self.modulation, dtype=x.dtype, device=x.device).unsqueeze(0) + e.unsqueeze(2)).unbind(2)
- x = (self.head(torch.addcmul(repeat_e(e[0], x), self.norm(x), 1 + repeat_e(e[1], x))))
+ x = self.head(modulate(x, self.norm, e[0], e[1]))
return x
@@ -1477,7 +1487,10 @@ class WanT2VCrossAttentionGather(WanSelfAttention):
# Handle video spatial structure
q = q.reshape(k.shape[0], -1, n, d).transpose(1, 2)
- x = optimized_attention(q, k, v, heads=self.num_heads, skip_reshape=True, skip_output_reshape=True, transformer_options=transformer_options)
+ q = AttentionTensorContainer(q)
+ k = AttentionTensorContainer(k)
+ v = AttentionTensorContainer(v)
+ x = optimized_attention(q, k, v, heads=self.num_heads, preferred_attention=self.comfy_attention, skip_reshape=True, skip_output_reshape=True, transformer_options=transformer_options)
x = x.transpose(1, 2).reshape(b, -1, n * d)
x = self.o(x)
@@ -1536,7 +1549,7 @@ class WanAttentionBlockAudio(WanAttentionBlock):
# self-attention
y = self.self_attn(
- torch.addcmul(repeat_e(e[0], x), self.norm1(x), 1 + repeat_e(e[1], x)),
+ modulate(x, self.norm1, e[0], e[1]),
freqs, transformer_options=transformer_options)
x = torch.addcmul(x, y, repeat_e(e[2], x))
@@ -1545,7 +1558,7 @@ class WanAttentionBlockAudio(WanAttentionBlock):
x = x + self.cross_attn(self.norm3(x), context, context_img_len=context_img_len, transformer_options=transformer_options)
if audio is not None:
x = self.audio_cross_attn_wrapper(x, audio, transformer_options=transformer_options)
- y = self.ffn(torch.addcmul(repeat_e(e[3], x), self.norm2(x), 1 + repeat_e(e[4], x)))
+ y = self.ffn(modulate(x, self.norm2, e[3], e[4]))
x = torch.addcmul(x, y, repeat_e(e[5], x))
return x
diff --git a/comfy/ldm/wan/model_animate.py b/comfy/ldm/wan/model_animate.py
index 9ebe5694b..91390128b 100644
--- a/comfy/ldm/wan/model_animate.py
+++ b/comfy/ldm/wan/model_animate.py
@@ -5,7 +5,7 @@ from einops import rearrange
import torch.nn.functional as F
import math
from .model import WanModel, sinusoidal_embedding_1d
-from comfy.ldm.modules.attention import optimized_attention
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention
import comfy.model_management
class CausalConv1d(nn.Module):
@@ -148,6 +148,7 @@ class FaceBlock(nn.Module):
):
factory_kwargs = {"device": device, "dtype": dtype}
super().__init__()
+ self.comfy_attention = ComfyAttention()
self.deterministic = False
self.hidden_size = hidden_size
@@ -201,7 +202,11 @@ class FaceBlock(nn.Module):
q = rearrange(q, "B (L S) H D -> (B L) S (H D)", L=T_comp)
- attn = optimized_attention(q, k, v, heads=self.heads_num)
+ q = AttentionTensorContainer(q)
+ k = AttentionTensorContainer(k)
+ v = AttentionTensorContainer(v)
+ del kv
+ attn = optimized_attention(q, k, v, heads=self.heads_num, preferred_attention=self.comfy_attention)
attn = rearrange(attn, "(B L) S C -> B (L S) C", L=T_comp)
diff --git a/comfy/ldm/wan/model_animate2.py b/comfy/ldm/wan/model_animate2.py
index 8bc55f4e9..cdcc1fbd7 100644
--- a/comfy/ldm/wan/model_animate2.py
+++ b/comfy/ldm/wan/model_animate2.py
@@ -14,9 +14,9 @@ import comfy.model_management
import comfy.quant_ops
import comfy.utils
from comfy.ldm.flux.math import apply_rope1
-from comfy.ldm.modules.attention import optimized_attention
+from comfy.ldm.modules.attention import AttentionTensorContainer, optimized_attention
-from .model import WanAttentionBlock, WanModel, WanSelfAttention, repeat_e, sinusoidal_embedding_1d
+from .model import WanAttentionBlock, WanModel, WanSelfAttention, modulate, repeat_e, sinusoidal_embedding_1d
class WanAnimate2SelfAttention(WanSelfAttention):
@@ -39,7 +39,7 @@ class WanAnimate2SelfAttention(WanSelfAttention):
def forward_pose(self, x, freqs, transformer_options={}):
b, s, n, d = *x.shape[:2], self.num_heads, self.head_dim
q, k, v = self.qkv(x, freqs)
- out = optimized_attention(q.reshape(b, s, n * d), k.reshape(b, s, n * d), v.reshape(b, s, n * d), heads=self.num_heads, transformer_options=transformer_options)
+ out = optimized_attention(AttentionTensorContainer(q.reshape(b, s, n * d)), AttentionTensorContainer(k.reshape(b, s, n * d)), AttentionTensorContainer(v.reshape(b, s, n * d)), heads=self.num_heads, preferred_attention=self.comfy_attention, transformer_options=transformer_options)
return self.o(self._attn1_patch(out, q, k, transformer_options)), k, v
def forward_gen(self, x, freqs, k_pose, v_pose, f_gen, hw, buffers, ref_strength=1.0, transformer_options={}):
@@ -50,7 +50,7 @@ class WanAnimate2SelfAttention(WanSelfAttention):
v[:, :hw] *= ref_strength # frame 0 is the reference image's slot
if k_pose is None: # pose influence windowed out: plain self-attention, no per-frame loop
- out = optimized_attention(q.reshape(b, s, n * d), k.reshape(b, s, n * d), v.reshape(b, s, n * d), heads=self.num_heads, transformer_options=transformer_options)
+ out = optimized_attention(AttentionTensorContainer(q.reshape(b, s, n * d)), AttentionTensorContainer(k.reshape(b, s, n * d)), AttentionTensorContainer(v.reshape(b, s, n * d)), heads=self.num_heads, preferred_attention=self.comfy_attention, transformer_options=transformer_options)
return self.o(self._attn1_patch(out, q, k, transformer_options))
# gen half is the same every frame; only the hw-token pose tail is rewritten
@@ -66,7 +66,7 @@ class WanAnimate2SelfAttention(WanSelfAttention):
kbuf[:, s:] = k_pose[:, (j - 1) * hw:j * hw]
vbuf[:, s:] = v_pose[:, (j - 1) * hw:j * hw]
kk, vv = kbuf, vbuf
- out[:, j * hw:(j + 1) * hw] = optimized_attention(q_j, kk.reshape(b, kk.shape[1], n * d), vv.reshape(b, kk.shape[1], n * d), heads=self.num_heads, transformer_options=transformer_options)
+ out[:, j * hw:(j + 1) * hw] = optimized_attention(AttentionTensorContainer(q_j), AttentionTensorContainer(kk.reshape(b, kk.shape[1], n * d)), AttentionTensorContainer(vv.reshape(b, kk.shape[1], n * d)), heads=self.num_heads, preferred_attention=self.comfy_attention, transformer_options=transformer_options)
return self.o(self._attn1_patch(out, q, k, transformer_options))
@@ -85,13 +85,13 @@ class WanAnimate2Block(WanAttentionBlock):
x = x + self.cross_attn(self.norm3(x), context, context_img_len=context_img_len, transformer_options=transformer_options)
for p in transformer_options.get("patches", {}).get("attn2_patch", []):
x = p({"x": x, "transformer_options": transformer_options})
- y = self.ffn(torch.addcmul(repeat_e(e[3], x), self.norm2(x), 1 + repeat_e(e[4], x)))
+ y = self.ffn(modulate(x, self.norm2, e[3], e[4]))
return torch.addcmul(x, y, repeat_e(e[5], x))
def forward_pose(self, x, e, freqs, context, context_img_len=257, transformer_options={}):
e = self._modulation(e, x)
x = x.contiguous()
- y, k, v = self.self_attn.forward_pose(torch.addcmul(repeat_e(e[0], x), self.norm1(x), 1 + repeat_e(e[1], x)), freqs, transformer_options=transformer_options)
+ y, k, v = self.self_attn.forward_pose(modulate(x, self.norm1, e[0], e[1]), freqs, transformer_options=transformer_options)
x = torch.addcmul(x, y, repeat_e(e[2], x))
del y
return self._cross_attn_ffn(x, e, context, context_img_len, transformer_options), k, v
@@ -99,12 +99,12 @@ class WanAnimate2Block(WanAttentionBlock):
def kv_from_input(self, x_pose, e, freqs, transformer_options={}):
e = self._modulation(e, x_pose)
x_pose = x_pose.contiguous()
- return self.self_attn.kv(torch.addcmul(repeat_e(e[0], x_pose), self.norm1(x_pose), 1 + repeat_e(e[1], x_pose)), freqs)
+ return self.self_attn.kv(modulate(x_pose, self.norm1, e[0], e[1]), freqs)
def forward_gen(self, x, e, freqs, context, k_pose, v_pose, f_gen, hw, buffers, ref_strength=1.0, context_img_len=257, transformer_options={}):
e = self._modulation(e, x)
x = x.contiguous()
- y = self.self_attn.forward_gen(torch.addcmul(repeat_e(e[0], x), self.norm1(x), 1 + repeat_e(e[1], x)), freqs, k_pose, v_pose, f_gen, hw, buffers, ref_strength=ref_strength, transformer_options=transformer_options)
+ y = self.self_attn.forward_gen(modulate(x, self.norm1, e[0], e[1]), freqs, k_pose, v_pose, f_gen, hw, buffers, ref_strength=ref_strength, transformer_options=transformer_options)
x = torch.addcmul(x, y, repeat_e(e[2], x))
del y
return self._cross_attn_ffn(x, e, context, context_img_len, transformer_options)
diff --git a/comfy/ldm/wan/model_multitalk.py b/comfy/ldm/wan/model_multitalk.py
index c9dd98c4d..951beb2e8 100644
--- a/comfy/ldm/wan/model_multitalk.py
+++ b/comfy/ldm/wan/model_multitalk.py
@@ -1,7 +1,7 @@
import torch
from einops import rearrange, repeat
import comfy
-from comfy.ldm.modules.attention import optimized_attention
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention
def calculate_x_ref_attn_map(visual_q, ref_k, ref_target_masks, split_num=8):
@@ -171,6 +171,7 @@ class SingleStreamAttention(torch.nn.Module):
) -> None:
super().__init__()
self.dim = dim
+ self.comfy_attention = ComfyAttention()
self.encoder_hidden_states_dim = encoder_hidden_states_dim
self.num_heads = num_heads
self.head_dim = dim // num_heads
@@ -203,11 +204,11 @@ class SingleStreamAttention(torch.nn.Module):
encoder_k, encoder_v = kv.view(B * N_t, encoder_hidden_states.shape[1], 2, self.num_heads, self.head_dim).unbind(2)
#print("q.shape", q.shape) #torch.Size([21, 1024, 40, 128])
- x = optimized_attention(
- q.transpose(1, 2),
- encoder_k.transpose(1, 2),
- encoder_v.transpose(1, 2),
- heads=self.num_heads, skip_reshape=True, skip_output_reshape=True).transpose(1, 2)
+ q = AttentionTensorContainer(q.transpose(1, 2))
+ encoder_k = AttentionTensorContainer(encoder_k.transpose(1, 2))
+ encoder_v = AttentionTensorContainer(encoder_v.transpose(1, 2))
+ del kv
+ x = optimized_attention(q, encoder_k, encoder_v, heads=self.num_heads, preferred_attention=self.comfy_attention, skip_reshape=True, skip_output_reshape=True).transpose(1, 2)
# linear transform
x = self.proj(x.reshape(B * N_t, S, self.dim))
@@ -324,11 +325,11 @@ class SingleStreamMultiAttention(SingleStreamAttention):
encoder_k = rearrange(encoder_k, "B H M K -> B M H K")
encoder_v = rearrange(encoder_v, "B H M K -> B M H K")
- x = optimized_attention(
- q.transpose(1, 2),
- encoder_k.transpose(1, 2),
- encoder_v.transpose(1, 2),
- heads=self.num_heads, skip_reshape=True, skip_output_reshape=True).transpose(1, 2)
+ q = AttentionTensorContainer(q.transpose(1, 2))
+ encoder_k = AttentionTensorContainer(encoder_k.transpose(1, 2))
+ encoder_v = AttentionTensorContainer(encoder_v.transpose(1, 2))
+ del encoder_kv
+ x = optimized_attention(q, encoder_k, encoder_v, heads=self.num_heads, preferred_attention=self.comfy_attention, skip_reshape=True, skip_output_reshape=True).transpose(1, 2)
# Linear projection
x = x.reshape(B, N, C)
diff --git a/comfy/ldm/wan/model_wandancer.py b/comfy/ldm/wan/model_wandancer.py
index aeec1d725..6de415c84 100644
--- a/comfy/ldm/wan/model_wandancer.py
+++ b/comfy/ldm/wan/model_wandancer.py
@@ -1,7 +1,7 @@
import torch
import torch.nn as nn
import comfy
-from comfy.ldm.modules.attention import optimized_attention
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention
from comfy.ldm.flux.math import apply_rope1
from comfy.ldm.flux.layers import EmbedND
@@ -12,6 +12,7 @@ class MusicSelfAttention(nn.Module):
def __init__(self, dim, num_heads, device=None, dtype=None, operations=None):
assert dim % num_heads == 0
super().__init__()
+ self.comfy_attention = ComfyAttention()
self.embed_dim = dim
self.num_heads = num_heads
self.head_dim = dim // num_heads
@@ -30,11 +31,13 @@ class MusicSelfAttention(nn.Module):
k = self.k_proj(x).view(b, s, n, d)
k = apply_rope1(k, freqs)
+ q = AttentionTensorContainer(q.view(b, s, n * d))
+ k = AttentionTensorContainer(k.view(b, s, n * d))
+ v = AttentionTensorContainer(self.v_proj(x).view(b, s, n * d))
x = optimized_attention(
- q.view(b, s, n * d),
- k.view(b, s, n * d),
- self.v_proj(x).view(b, s, n * d),
+ q, k, v,
heads=self.num_heads,
+ preferred_attention=self.comfy_attention,
)
return self.out_proj(x)
diff --git a/tests-unit/comfy_test/test_sensenova.py b/tests-unit/comfy_test/test_sensenova.py
index 4e2398e41..505080e27 100644
--- a/tests-unit/comfy_test/test_sensenova.py
+++ b/tests-unit/comfy_test/test_sensenova.py
@@ -584,6 +584,7 @@ def test_sensenova_prefix_mask_matches_attention_dtype(monkeypatch):
attention = SimpleNamespace(
_project=lambda hidden_states, rope, generation: (query, key, value),
o_proj=lambda output: output,
+ comfy_attention=sensenova_model.ComfyAttention(),
)
mask = torch.zeros(1, 1, 3, 3, dtype=torch.bfloat16)