The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Add comfy_attention and AttentionTensorContainer support to more models. (#16766)
comfyanonymousb87fe48b0491
diff --git a/comfy/ldm/ace/ace_step15.py b/comfy/ldm/ace/ace_step15.pyindex 02182c49f..97f90ffc1 100644--- a/comfy/ldm/ace/ace_step15.py+++ b/comfy/ldm/ace/ace_step15.py@@ -3,7 +3,7 @@ import torchimport torch.nn as nnimport torch.nn.functional as Fimport itertools-from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attentionimport comfy.model_managementfrom comfy.ldm.flux.layers import timestep_embedding@@ -159,6 +159,7 @@ class AceStepAttention(nn.Module):operations=None):super().__init__()+ self.comfy_attention = ComfyAttention()self.hidden_size = hidden_sizeself.num_heads = num_headsself.num_kv_heads = num_kv_heads@@ -241,7 +242,8 @@ class AceStepAttention(nn.Module):else:attn_bias = window_bias- attn_output = optimized_attention(query_states, key_states, value_states, self.num_heads, attn_bias, skip_reshape=True, low_precision_attention=False, **gqa_kwargs)+ query_states, key_states, value_states = AttentionTensorContainer(query_states), AttentionTensorContainer(key_states), AttentionTensorContainer(value_states)+ attn_output = optimized_attention(query_states, key_states, value_states, self.num_heads, attn_bias, skip_reshape=True, low_precision_attention=False, preferred_attention=self.comfy_attention, **gqa_kwargs)attn_output = self.o_proj(attn_output)return attn_outputdiff --git a/comfy/ldm/ace/attention.py b/comfy/ldm/ace/attention.pyindex 670eb9783..3d0ec5b95 100644--- a/comfy/ldm/ace/attention.py+++ b/comfy/ldm/ace/attention.py@@ -19,7 +19,7 @@ import torch.nn.functional as Ffrom torch import nnimport comfy.model_management-from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attentionclass Attention(nn.Module):def __init__(@@ -50,6 +50,7 @@ class Attention(nn.Module):dtype=None, device=None, operations=None):super().__init__()+ self.comfy_attention = ComfyAttention()self.inner_dim = out_dim if out_dim is not None else dim_head * headsself.inner_kv_dim = self.inner_dim if kv_heads is None else dim_head * kv_heads@@ -435,9 +436,11 @@ class CustomerAttnProcessor2_0:attention_mask = attention_mask.view(batch_size, attn.heads, -1, attention_mask.shape[-1])# the output of sdp = (batch, num_heads, seq_len, head_dim)+ heads = query.shape[1]+ query, key, value = AttentionTensorContainer(query), AttentionTensorContainer(key), AttentionTensorContainer(value)hidden_states = optimized_attention(- query, key, value, heads=query.shape[1], mask=attention_mask, skip_reshape=True, transformer_options=transformer_options,- ).to(query.dtype)+ query, key, value, heads=heads, mask=attention_mask, skip_reshape=True, preferred_attention=attn.comfy_attention, transformer_options=transformer_options,+ )# linear projhidden_states = attn.to_out[0](hidden_states)diff --git a/comfy/ldm/aura/mmdit.py b/comfy/ldm/aura/mmdit.pyindex 66d9613b6..5f5128724 100644--- a/comfy/ldm/aura/mmdit.py+++ b/comfy/ldm/aura/mmdit.py@@ -7,7 +7,7 @@ import torchimport torch.nn as nnimport torch.nn.functional as F-from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attentionimport comfy.opsimport comfy.patcher_extensionimport comfy.ldm.common_dit@@ -63,6 +63,7 @@ class MultiHeadLayerNorm(nn.Module):class SingleAttention(nn.Module):def __init__(self, dim, n_heads, mh_qknorm=False, dtype=None, device=None, operations=None):super().__init__()+ self.comfy_attention = ComfyAttention()self.n_heads = n_headsself.head_dim = dim // n_heads@@ -95,7 +96,8 @@ class SingleAttention(nn.Module):v = v.view(bsz, seqlen1, self.n_heads, self.head_dim)q, k = self.q_norm1(q), self.k_norm1(k)- output = optimized_attention(q.permute(0, 2, 1, 3), k.permute(0, 2, 1, 3), v.permute(0, 2, 1, 3), self.n_heads, skip_reshape=True, transformer_options=transformer_options)+ q, k, v = AttentionTensorContainer(q.permute(0, 2, 1, 3)), AttentionTensorContainer(k.permute(0, 2, 1, 3)), AttentionTensorContainer(v.permute(0, 2, 1, 3))+ output = optimized_attention(q, k, v, self.n_heads, skip_reshape=True, preferred_attention=self.comfy_attention, transformer_options=transformer_options)c = self.w1o(output)return c@@ -104,6 +106,7 @@ class SingleAttention(nn.Module):class DoubleAttention(nn.Module):def __init__(self, dim, n_heads, mh_qknorm=False, dtype=None, device=None, operations=None):super().__init__()+ self.comfy_attention = ComfyAttention()self.n_heads = n_headsself.head_dim = dim // n_heads@@ -168,7 +171,9 @@ class DoubleAttention(nn.Module):torch.cat([cv, xv], dim=1),)- output = optimized_attention(q.permute(0, 2, 1, 3), k.permute(0, 2, 1, 3), v.permute(0, 2, 1, 3), self.n_heads, skip_reshape=True, transformer_options=transformer_options)+ del cq, ck, cv, xq, xk, xv+ q, k, v = AttentionTensorContainer(q.permute(0, 2, 1, 3)), AttentionTensorContainer(k.permute(0, 2, 1, 3)), AttentionTensorContainer(v.permute(0, 2, 1, 3))+ output = optimized_attention(q, k, v, self.n_heads, skip_reshape=True, preferred_attention=self.comfy_attention, transformer_options=transformer_options)c, x = output.split([seqlen1, seqlen2], dim=1)c = self.w1o(c)diff --git a/comfy/ldm/boogu/model.py b/comfy/ldm/boogu/model.pyindex ca88bdeb1..4dd4ff414 100644--- a/comfy/ldm/boogu/model.py+++ b/comfy/ldm/boogu/model.py@@ -12,7 +12,7 @@ from einops import rearrangeimport comfy.ldm.common_ditimport comfy.ldm.omnigen.omnigen2-from comfy.ldm.modules.attention import optimized_attention_masked+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention_maskedfrom comfy.ldm.omnigen.omnigen2 import (OmniGen2RotaryPosEmbed,Lumina2CombinedTimestepCaptionEmbedding,@@ -58,6 +58,7 @@ class BooguDoubleStreamProcessor(nn.Module):query = torch.cat([instruct_q, img_q], dim=1)key = torch.cat([instruct_k, img_k], dim=1)value = torch.cat([instruct_v, img_v], dim=1)+ del instruct_q, instruct_k, instruct_v, img_q, img_k, img_vquery = query.view(batch_size, -1, attn.heads, attn.dim_head)key = key.view(batch_size, -1, attn.kv_heads, attn.dim_head)@@ -70,12 +71,12 @@ class BooguDoubleStreamProcessor(nn.Module):query = apply_rotary_emb(query, rotary_emb)key = apply_rotary_emb(key, rotary_emb)- query = query.transpose(1, 2)- key = key.transpose(1, 2)- value = value.transpose(1, 2)+ query = AttentionTensorContainer(query.transpose(1, 2))+ key = AttentionTensorContainer(key.transpose(1, 2))+ value = AttentionTensorContainer(value.transpose(1, 2))gqa_kwargs = {"enable_gqa": True} if attn.kv_heads < attn.heads else {}- hidden_states = optimized_attention_masked(query, key, value, attn.heads, attention_mask, skip_reshape=True, transformer_options=transformer_options, **gqa_kwargs)+ hidden_states = optimized_attention_masked(query, key, value, attn.heads, attention_mask, skip_reshape=True, preferred_attention=attn.comfy_attention, transformer_options=transformer_options, **gqa_kwargs)# Split back to instruction/image, apply per-stream output projections, recombine.instruct_hidden_states = self.instruct_out(hidden_states[:, :L_instruct])@@ -90,6 +91,7 @@ class BooguJointAttention(nn.Module):# Holds the shared q/k RMSNorm + final output projectiondef __init__(self, dim, head_dim, heads, kv_heads, eps=1e-5, dtype=None, device=None, operations=None):super().__init__()+ self.comfy_attention = ComfyAttention()self.heads = headsself.kv_heads = kv_headsself.dim_head = head_dimdiff --git a/comfy/ldm/ernie/model.py b/comfy/ldm/ernie/model.pyindex 88a3775d0..678e4c99a 100644--- a/comfy/ldm/ernie/model.py+++ b/comfy/ldm/ernie/model.py@@ -3,7 +3,7 @@ import torchimport torch.nn as nnimport torch.nn.functional as F-from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attentionimport comfy.model_managementimport comfy.opsimport comfy.quant_ops@@ -86,6 +86,7 @@ class TimestepEmbedding(nn.Module):class ErnieImageAttention(nn.Module):def __init__(self, query_dim: int, heads: int, dim_head: int, eps: float = 1e-6, operations=None, device=None, dtype=None):super().__init__()+ self.comfy_attention = ComfyAttention()self.heads = headsself.head_dim = dim_headself.inner_dim = heads * dim_head@@ -124,10 +125,12 @@ class ErnieImageAttention(nn.Module):if image_rotary_emb is not None:query, key = comfy.quant_ops.ck.apply_rope_split_half(query, key, image_rotary_emb)- q_flat = query.reshape(B, S, -1)- k_flat = key.reshape(B, S, -1)+ q_flat = AttentionTensorContainer(query.reshape(B, S, -1))+ k_flat = AttentionTensorContainer(key.reshape(B, S, -1))+ v_flat = AttentionTensorContainer(v_flat)+ del query, key- hidden_states = optimized_attention(q_flat, k_flat, v_flat, self.heads, mask=attention_mask)+ hidden_states = optimized_attention(q_flat, k_flat, v_flat, self.heads, mask=attention_mask, preferred_attention=self.comfy_attention)return self.to_out[0](hidden_states)diff --git a/comfy/ldm/hydit/attn_layers.py b/comfy/ldm/hydit/attn_layers.pyindex 3ca25a5df..192cce591 100644--- a/comfy/ldm/hydit/attn_layers.py+++ b/comfy/ldm/hydit/attn_layers.py@@ -1,7 +1,7 @@import torchimport torch.nn as nnfrom typing import Tuple, Union, Optional-from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attentiondef reshape_for_broadcast(freqs_cis: Union[torch.Tensor, Tuple[torch.Tensor]], x: torch.Tensor, head_first=False):@@ -113,6 +113,7 @@ class CrossAttention(nn.Module):factory_kwargs = {'device': device, 'dtype': dtype}super().__init__()self.attn_precision = attn_precision+ self.comfy_attention = ComfyAttention()self.qdim = qdimself.kdim = kdimself.num_heads = num_heads@@ -156,12 +157,15 @@ class CrossAttention(nn.Module):qq, _ = apply_rotary_emb(q, None, freqs_cis_img)assert qq.shape == q.shape, f'qq: {qq.shape}, q: {q.shape}'q = qq+ del qqq = q.transpose(-2, -3).contiguous() # q -> B, L1, H, C - B, H, L1, Ck = k.transpose(-2, -3).contiguous() # k -> B, L2, H, C - B, H, C, L2v = v.transpose(-2, -3).contiguous()- context = optimized_attention(q, k, v, self.num_heads, skip_reshape=True, attn_precision=self.attn_precision)+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)+ del kv+ context = optimized_attention(q, k, v, self.num_heads, skip_reshape=True, attn_precision=self.attn_precision, preferred_attention=self.comfy_attention)out = self.out_proj(context) # context.reshape - B, L1, -1out = self.proj_drop(out)@@ -178,6 +182,7 @@ class Attention(nn.Module):def __init__(self, dim, num_heads, qkv_bias=True, qk_norm=False, attn_drop=0., proj_drop=0., attn_precision=None, dtype=None, device=None, operations=None):super().__init__()self.attn_precision = attn_precision+ self.comfy_attention = ComfyAttention()self.dim = dimself.num_heads = num_headsassert self.dim % num_heads == 0, 'dim should be divisible by num_heads'@@ -208,8 +213,11 @@ class Attention(nn.Module):assert qq.shape == q.shape and kk.shape == k.shape, \f'qq: {qq.shape}, q: {q.shape}, kk: {kk.shape}, k: {k.shape}'q, k = qq, kk+ del qq, kk- x = optimized_attention(q, k, v, self.num_heads, skip_reshape=True, attn_precision=self.attn_precision)+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)+ del qkv+ x = optimized_attention(q, k, v, self.num_heads, skip_reshape=True, attn_precision=self.attn_precision, preferred_attention=self.comfy_attention)x = self.out_proj(x)x = self.proj_drop(x)diff --git a/comfy/ldm/hydit/poolers.py b/comfy/ldm/hydit/poolers.pyindex c1b878ed6..db685ac23 100644--- a/comfy/ldm/hydit/poolers.py+++ b/comfy/ldm/hydit/poolers.py@@ -1,11 +1,12 @@import torchimport torch.nn as nn-from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attentionimport comfy.opsclass AttentionPool(nn.Module):def __init__(self, spacial_dim: int, embed_dim: int, num_heads: int, output_dim: int = None, dtype=None, device=None, operations=None):super().__init__()+ self.comfy_attention = ComfyAttention()self.positional_embedding = nn.Parameter(torch.empty(spacial_dim + 1, embed_dim, dtype=dtype, device=device))self.k_proj = operations.Linear(embed_dim, embed_dim, dtype=dtype, device=device)self.q_proj = operations.Linear(embed_dim, embed_dim, dtype=dtype, device=device)@@ -30,7 +31,8 @@ class AttentionPool(nn.Module):k = k.view(k.shape[0], batch_size * self.num_heads, head_dim).transpose(0, 1).view(batch_size, self.num_heads, -1, head_dim)v = v.view(v.shape[0], batch_size * self.num_heads, head_dim).transpose(0, 1).view(batch_size, self.num_heads, -1, head_dim)- attn_output = optimized_attention(q, k, v, self.num_heads, skip_reshape=True).transpose(0, 1)+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)+ attn_output = optimized_attention(q, k, v, self.num_heads, skip_reshape=True, preferred_attention=self.comfy_attention).transpose(0, 1)attn_output = self.c_proj(attn_output)return attn_output.squeeze(0)diff --git a/comfy/ldm/lens/model.py b/comfy/ldm/lens/model.pyindex cd5015ddc..07b9194a6 100644--- a/comfy/ldm/lens/model.py+++ b/comfy/ldm/lens/model.py@@ -12,7 +12,7 @@ import comfy.ldm.flux.layersimport comfy.patcher_extensionfrom comfy.ldm.flux.layers import EmbedNDfrom comfy.ldm.flux.math import apply_rope-from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attentiondef _lens_time_proj(t: torch.Tensor, dim: int = 256) -> torch.Tensor:@@ -105,6 +105,7 @@ class LensJointAttention(nn.Module):operations=None,) -> None:super().__init__()+ self.comfy_attention = ComfyAttention()self.inner_dim = out_dim if out_dim is not None else dim_head * headsself.heads = self.inner_dim // dim_headself.dim_head = dim_head@@ -148,6 +149,7 @@ class LensJointAttention(nn.Module):txt_q, txt_k, txt_v = txt_qkv.unbind(dim=2)txt_q = self.norm_added_q(txt_q)txt_k = self.norm_added_k(txt_k)+ del txt_qkv# [B, S, H, D] → [B, H, S, D] for attention, dels to avoid VRAM peaksq = torch.cat([img_q, txt_q], dim=1).transpose(1, 2)@@ -167,9 +169,11 @@ class LensJointAttention(nn.Module):)attention_mask = attention_mask.to(q.dtype)+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)out = optimized_attention(q, k, v, self.heads, mask=attention_mask, skip_reshape=True,transformer_options=transformer_options,+ preferred_attention=self.comfy_attention,)img_out = self.to_out[1](self.to_out[0](out[:, :seq_img, :]))diff --git a/comfy/ldm/minimax_music/dit.py b/comfy/ldm/minimax_music/dit.pyindex 211e0d7db..258e19b85 100644--- a/comfy/ldm/minimax_music/dit.py+++ b/comfy/ldm/minimax_music/dit.py@@ -6,7 +6,7 @@ from torch import nnimport comfy.model_managementimport comfy.opsimport comfy.quant_ops-from comfy.ldm.modules.attention import optimized_attention_for_device+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention_for_deviceMAX_CONDITION_FRAMES = 200@@ -66,6 +66,7 @@ def _apply_rope(x, rotation_matrix):class Attention(nn.Module):def __init__(self, dim, dim_heads, dtype, device, operations):super().__init__()+ self.comfy_attention = ComfyAttention()self.num_heads = dim // dim_headsself.dim_heads = dim_headsself.to_qkv = operations.Linear(dim, dim * 3, bias=False, dtype=dtype, device=device)@@ -85,8 +86,10 @@ class Attention(nn.Module):rotated_q, rotated_k = comfy.quant_ops.ck.apply_rope_split_half(q[..., :rotary_dims], k[..., :rotary_dims], rotation_matrix)q = torch.cat((rotated_q, q[..., rotary_dims:]), dim=-1)k = torch.cat((rotated_k, k[..., rotary_dims:]), dim=-1)+ del rotated_q, rotated_kattention = optimized_attention_for_device(q.device)- out = attention(q, k, v, self.num_heads, skip_reshape=True)+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)+ out = attention(q, k, v, self.num_heads, skip_reshape=True, preferred_attention=self.comfy_attention)return self.to_out(out)diff --git a/comfy/ldm/omnigen/omnigen2.py b/comfy/ldm/omnigen/omnigen2.pyindex d18a9f6d0..b265edd16 100644--- a/comfy/ldm/omnigen/omnigen2.py+++ b/comfy/ldm/omnigen/omnigen2.py@@ -9,7 +9,7 @@ from einops import rearrange, repeatfrom comfy.ldm.lightricks.model import Timestepsfrom comfy.ldm.flux.layers import EmbedNDfrom comfy.ldm.flux.math import apply_rope1-from comfy.ldm.modules.attention import optimized_attention_masked+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention_maskedimport comfy.model_managementimport comfy.ldm.common_dit@@ -102,6 +102,7 @@ class Lumina2CombinedTimestepCaptionEmbedding(nn.Module):class Attention(nn.Module):def __init__(self, query_dim: int, dim_head: int, heads: int, kv_heads: int, eps: float = 1e-5, bias: bool = False, dtype=None, device=None, operations=None):super().__init__()+ self.comfy_attention = ComfyAttention()self.heads = headsself.kv_heads = kv_headsself.dim_head = dim_head@@ -137,12 +138,12 @@ class Attention(nn.Module):query = apply_rotary_emb(query, image_rotary_emb)key = apply_rotary_emb(key, image_rotary_emb)- query = query.transpose(1, 2)- key = key.transpose(1, 2)- value = value.transpose(1, 2)+ query = AttentionTensorContainer(query.transpose(1, 2))+ key = AttentionTensorContainer(key.transpose(1, 2))+ value = AttentionTensorContainer(value.transpose(1, 2))gqa_kwargs = {"enable_gqa": True} if self.kv_heads < self.heads else {}- hidden_states = optimized_attention_masked(query, key, value, self.heads, attention_mask, skip_reshape=True, transformer_options=transformer_options, **gqa_kwargs)+ hidden_states = optimized_attention_masked(query, key, value, self.heads, attention_mask, skip_reshape=True, preferred_attention=self.comfy_attention, transformer_options=transformer_options, **gqa_kwargs)hidden_states = self.to_out[0](hidden_states)return hidden_statesdiff --git a/comfy/ldm/pixart/blocks.py b/comfy/ldm/pixart/blocks.pyindex 2225076e5..b27a66186 100644--- a/comfy/ldm/pixart/blocks.py+++ b/comfy/ldm/pixart/blocks.py@@ -7,7 +7,7 @@ import torch.nn.functional as Ffrom einops import rearrangefrom comfy.ldm.modules.diffusionmodules.mmdit import TimestepEmbedder, Mlp, timestep_embedding-from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention# if model_management.xformers_enabled():# import xformers.ops@@ -25,6 +25,7 @@ def t2i_modulate(x, shift, scale):class MultiHeadCrossAttention(nn.Module):def __init__(self, d_model, num_heads, attn_drop=0., proj_drop=0., dtype=None, device=None, operations=None, **kwargs):super(MultiHeadCrossAttention, self).__init__()+ self.comfy_attention = ComfyAttention()assert d_model % num_heads == 0, "d_model must be divisible by num_heads"self.d_model = d_model@@ -71,7 +72,9 @@ class MultiHeadCrossAttention(nn.Module):# attn_mask = torch.block_diag(attn_mask, attn_mask_template)# x = optimized_attention(q, k, v, self.num_heads, mask=attn_mask, skip_reshape=True)- x = optimized_attention(q.view(B, -1, C), k.view(B, -1, C), v.view(B, -1, C), self.num_heads, mask=None)+ q, k, v = AttentionTensorContainer(q.view(B, -1, C)), AttentionTensorContainer(k.view(B, -1, C)), AttentionTensorContainer(v.view(B, -1, C))+ del kv+ x = optimized_attention(q, k, v, self.num_heads, mask=None, preferred_attention=self.comfy_attention)x = self.proj(x)x = self.proj_drop(x)return x@@ -87,6 +90,7 @@ class AttentionKVCompress(nn.Module):qkv_bias (bool: If True, add a learnable bias to query, key, value."""super().__init__()+ self.comfy_attention = ComfyAttention()assert dim % num_heads == 0, 'dim should be divisible by num_heads'self.num_heads = num_headsself.head_dim = dim // num_heads@@ -169,7 +173,9 @@ class AttentionKVCompress(nn.Module):# attention 2q, k, v = map(lambda t: t.transpose(1, 2), (q, k, v),)- x = optimized_attention(q, k, v, self.num_heads, mask=None, skip_reshape=True)+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)+ del qkv+ x = optimized_attention(q, k, v, self.num_heads, mask=None, skip_reshape=True, preferred_attention=self.comfy_attention)x = x.view(B, N, C)x = self.proj(x)diff --git a/comfy/ldm/pixeldit/model.py b/comfy/ldm/pixeldit/model.pyindex 3b30b9226..7f9758282 100644--- a/comfy/ldm/pixeldit/model.py+++ b/comfy/ldm/pixeldit/model.py@@ -6,7 +6,7 @@ import comfy.ldm.common_ditimport comfy.patcher_extensionfrom comfy.ldm.flux.math import apply_rope, ropefrom comfy.ldm.hidream.model import FeedForwardSwiGLU-from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attentionfrom comfy.ldm.modules.diffusionmodules.mmdit import TimestepEmbedderfrom .modules import (@@ -27,6 +27,7 @@ class MMDiTJointAttention(nn.Module):"""def __init__(self, dim, num_heads=8, qkv_bias=False, dtype=None, device=None, operations=None):super().__init__()+ self.comfy_attention = ComfyAttention()assert dim % num_heads == 0self.num_heads = num_headsself.head_dim = dim // num_heads@@ -66,10 +67,12 @@ class MMDiTJointAttention(nn.Module):k_joint = torch.cat([ky, kx], dim=2)v_joint = torch.cat([vy, vx], dim=2)+ del qkv_x, qkv_y, qx, kx, vx, qy, ky, vy+ q_joint, k_joint, v_joint = AttentionTensorContainer(q_joint), AttentionTensorContainer(k_joint), AttentionTensorContainer(v_joint)out_joint = optimized_attention(q_joint, k_joint, v_joint, H,mask=attn_mask, skip_reshape=True, skip_output_reshape=True,- transformer_options=transformer_options,+ transformer_options=transformer_options, preferred_attention=self.comfy_attention,)out_y = out_joint[:, :, :Ny, :].transpose(1, 2).reshape(B, Ny, H * D)diff --git a/comfy/ldm/pixeldit/modules.py b/comfy/ldm/pixeldit/modules.pyindex 4b1e538c7..88426e5a0 100644--- a/comfy/ldm/pixeldit/modules.py+++ b/comfy/ldm/pixeldit/modules.py@@ -2,7 +2,7 @@ import torchimport torch.nn as nnfrom comfy.ldm.flux.math import apply_rope, rope-from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attentionfrom comfy.ldm.modules.diffusionmodules.mmdit import Mlp, get_1d_sincos_pos_embed_from_grid_torch@@ -58,6 +58,7 @@ class RotaryAttention(nn.Module):"""Single-stream self-attention with rotary positional encoding (used inside PiTBlock)."""def __init__(self, dim, num_heads=8, qkv_bias=False, dtype=None, device=None, operations=None):super().__init__()+ self.comfy_attention = ComfyAttention()assert dim % num_heads == 0self.num_heads = num_headsself.head_dim = dim // num_heads@@ -73,7 +74,9 @@ class RotaryAttention(nn.Module):qkv = self.qkv(x).reshape(B, N, 3, H, D).permute(2, 0, 3, 1, 4)q, k, v = qkv.unbind(0)q, k = apply_rope(self.q_norm(q), self.k_norm(k), pos[None, None])- x = optimized_attention(q, k, v, H, mask=mask, skip_reshape=True, transformer_options=transformer_options)+ del qkv+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)+ x = optimized_attention(q, k, v, H, mask=mask, skip_reshape=True, preferred_attention=self.comfy_attention, transformer_options=transformer_options)return self.proj(x)diff --git a/comfy/ldm/qwen_image/model.py b/comfy/ldm/qwen_image/model.pyindex e49886dd9..b06a8fe0e 100644--- a/comfy/ldm/qwen_image/model.py+++ b/comfy/ldm/qwen_image/model.py@@ -6,7 +6,7 @@ from typing import Optional, Tuplefrom einops import repeatfrom comfy.ldm.lightricks.model import TimestepEmbedding, Timesteps-from comfy.ldm.modules.attention import optimized_attention_masked+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention_maskedfrom comfy.ldm.flux.layers import EmbedNDimport comfy.ldm.common_ditimport comfy.patcher_extension@@ -108,6 +108,7 @@ class Attention(nn.Module):operations=None):super().__init__()+ self.comfy_attention = ComfyAttention()self.inner_dim = out_dim if out_dim is not None else dim_head * headsself.inner_kv_dim = self.inner_dimself.heads = heads@@ -172,6 +173,7 @@ class Attention(nn.Module):joint_query = torch.cat([txt_query, img_query], dim=2)joint_key = torch.cat([txt_key, img_key], dim=2)joint_value = torch.cat([txt_value, img_value], dim=2)+ del img_query, img_key, img_value, txt_query, txt_key, txt_valueif encoder_hidden_states_mask is not None:attn_mask = torch.zeros((batch_size, 1, seq_txt + seq_img), dtype=hidden_states.dtype, device=hidden_states.device)@@ -179,7 +181,7 @@ class Attention(nn.Module):else:attn_mask = None- extra_options["img_slice"] = [txt_query.shape[2], joint_query.shape[2]]+ extra_options["img_slice"] = [seq_txt, joint_query.shape[2]]if "attn1_patch" in transformer_patches:patch = transformer_patches["attn1_patch"]for p in patch:@@ -188,10 +190,13 @@ class Attention(nn.Module):joint_query = apply_rope1(joint_query, image_rotary_emb)joint_key = apply_rope1(joint_key, image_rotary_emb)+ joint_query = AttentionTensorContainer(joint_query)+ joint_key = AttentionTensorContainer(joint_key)+ joint_value = AttentionTensorContainer(joint_value)joint_hidden_states = optimized_attention_masked(joint_query, joint_key, joint_value, self.heads,attn_mask, transformer_options=transformer_options,- skip_reshape=True)+ skip_reshape=True, preferred_attention=self.comfy_attention)txt_attn_output = joint_hidden_states[:, :seq_txt, :]img_attn_output = joint_hidden_states[:, seq_txt:, :]diff --git a/comfy/ldm/qwen_image21/model.py b/comfy/ldm/qwen_image21/model.pyindex b804b9eca..e50d6c3cb 100644--- a/comfy/ldm/qwen_image21/model.py+++ b/comfy/ldm/qwen_image21/model.py@@ -12,7 +12,7 @@ import comfy.rmsnormfrom comfy.ldm.flux.layers import EmbedND, timestep_embeddingfrom comfy.ldm.flux.math import apply_rope1from comfy.ldm.lightricks.model import TimestepEmbedding-from comfy.ldm.modules.attention import ComfyAttention, optimized_attention+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attentionfrom comfy.ldm.wan.model_animate2 import PoseBranchCache@@ -104,6 +104,7 @@ class Attention(nn.Module):q, k = comfy.quant_ops.ck.rms_rope(q, k, pe, q_scale, k_scale, self.norm_q.eps)comfy.ops.uncast_bias_weight(self.norm_q, q_scale, None, q_stream)comfy.ops.uncast_bias_weight(self.norm_k, k_scale, None, k_stream)+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)return self.to_out[0](attn_fn(q, k, v, self.heads, preferred_attention=self.comfy_attention))@@ -196,11 +197,18 @@ class QwenImage21FunControl(nn.Module):def block_causal_attention(segments, transformer_options={}, cache=None, block_index=0, prefix_len=0):# segments: (start, end, mask); text segments get a causal mask, image blocks attend to everything before their enddef attn(q, k, v, heads, preferred_attention=None):+ q, k, v = q.take(), k.take(), v.take()if cache is not None:# K and V stacked on dim 1 so batch stays first and quantized rows are per token and headcache.put(block_index, torch.stack([k[:, :prefix_len], v[:, :prefix_len]], dim=1))- outs = [optimized_attention(q[:, start:end].flatten(2), k[:, :end].flatten(2), v[:, :end].flatten(2), heads, mask=mask, transformer_options=transformer_options, preferred_attention=preferred_attention)- for start, end, mask in segments]+ outs = []+ for i, (start, end, mask) in enumerate(segments):+ query = AttentionTensorContainer(q[:, start:end].flatten(2))+ key = AttentionTensorContainer(k[:, :end].flatten(2))+ value = AttentionTensorContainer(v[:, :end].flatten(2))+ if i == len(segments) - 1:+ del q, k, v+ outs.append(optimized_attention(query, key, value, heads, mask=mask, transformer_options=transformer_options, preferred_attention=preferred_attention))return torch.cat(outs, dim=1) if len(outs) > 1 else outs[0]return attn@@ -208,7 +216,10 @@ def block_causal_attention(segments, transformer_options={}, cache=None, block_idef prefix_cached_attention(prefix_k, prefix_v, transformer_options={}):# target-only queries: block-causal reduces to full attention over [cached prefix, target]def attn(q, k, v, heads, preferred_attention=None):- return optimized_attention(q.flatten(2), torch.cat([prefix_k, k], dim=1).flatten(2), torch.cat([prefix_v, v], dim=1).flatten(2), heads, transformer_options=transformer_options, preferred_attention=preferred_attention)+ q = AttentionTensorContainer(q.take().flatten(2))+ k = AttentionTensorContainer(torch.cat([prefix_k, k.take()], dim=1).flatten(2))+ v = AttentionTensorContainer(torch.cat([prefix_v, v.take()], dim=1).flatten(2))+ return optimized_attention(q, k, v, heads, transformer_options=transformer_options, preferred_attention=preferred_attention)return attn