Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Add comfy_attention and AttentionTensorContainer support to more models. (#16766)

comfyanonymousb87fe48b0491
diff --git a/comfy/ldm/ace/ace_step15.py b/comfy/ldm/ace/ace_step15.py
index 02182c49f..97f90ffc1 100644
--- a/comfy/ldm/ace/ace_step15.py
+++ b/comfy/ldm/ace/ace_step15.py
@@ -3,7 +3,7 @@ import torch
import torch.nn as nn
import torch.nn.functional as F
import itertools
-from comfy.ldm.modules.attention import optimized_attention
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention
import comfy.model_management
from comfy.ldm.flux.layers import timestep_embedding
@@ -159,6 +159,7 @@ class AceStepAttention(nn.Module):
operations=None
):
super().__init__()
+ self.comfy_attention = ComfyAttention()
self.hidden_size = hidden_size
self.num_heads = num_heads
self.num_kv_heads = num_kv_heads
@@ -241,7 +242,8 @@ class AceStepAttention(nn.Module):
else:
attn_bias = window_bias
- attn_output = optimized_attention(query_states, key_states, value_states, self.num_heads, attn_bias, skip_reshape=True, low_precision_attention=False, **gqa_kwargs)
+ query_states, key_states, value_states = AttentionTensorContainer(query_states), AttentionTensorContainer(key_states), AttentionTensorContainer(value_states)
+ attn_output = optimized_attention(query_states, key_states, value_states, self.num_heads, attn_bias, skip_reshape=True, low_precision_attention=False, preferred_attention=self.comfy_attention, **gqa_kwargs)
attn_output = self.o_proj(attn_output)
return attn_output
diff --git a/comfy/ldm/ace/attention.py b/comfy/ldm/ace/attention.py
index 670eb9783..3d0ec5b95 100644
--- a/comfy/ldm/ace/attention.py
+++ b/comfy/ldm/ace/attention.py
@@ -19,7 +19,7 @@ import torch.nn.functional as F
from torch import nn
import comfy.model_management
-from comfy.ldm.modules.attention import optimized_attention
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention
class Attention(nn.Module):
def __init__(
@@ -50,6 +50,7 @@ class Attention(nn.Module):
dtype=None, device=None, operations=None
):
super().__init__()
+ self.comfy_attention = ComfyAttention()
self.inner_dim = out_dim if out_dim is not None else dim_head * heads
self.inner_kv_dim = self.inner_dim if kv_heads is None else dim_head * kv_heads
@@ -435,9 +436,11 @@ class CustomerAttnProcessor2_0:
attention_mask = attention_mask.view(batch_size, attn.heads, -1, attention_mask.shape[-1])
# the output of sdp = (batch, num_heads, seq_len, head_dim)
+ heads = query.shape[1]
+ query, key, value = AttentionTensorContainer(query), AttentionTensorContainer(key), AttentionTensorContainer(value)
hidden_states = optimized_attention(
- query, key, value, heads=query.shape[1], mask=attention_mask, skip_reshape=True, transformer_options=transformer_options,
- ).to(query.dtype)
+ query, key, value, heads=heads, mask=attention_mask, skip_reshape=True, preferred_attention=attn.comfy_attention, transformer_options=transformer_options,
+ )
# linear proj
hidden_states = attn.to_out[0](hidden_states)
diff --git a/comfy/ldm/aura/mmdit.py b/comfy/ldm/aura/mmdit.py
index 66d9613b6..5f5128724 100644
--- a/comfy/ldm/aura/mmdit.py
+++ b/comfy/ldm/aura/mmdit.py
@@ -7,7 +7,7 @@ import torch
import torch.nn as nn
import torch.nn.functional as F
-from comfy.ldm.modules.attention import optimized_attention
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention
import comfy.ops
import comfy.patcher_extension
import comfy.ldm.common_dit
@@ -63,6 +63,7 @@ class MultiHeadLayerNorm(nn.Module):
class SingleAttention(nn.Module):
def __init__(self, dim, n_heads, mh_qknorm=False, dtype=None, device=None, operations=None):
super().__init__()
+ self.comfy_attention = ComfyAttention()
self.n_heads = n_heads
self.head_dim = dim // n_heads
@@ -95,7 +96,8 @@ class SingleAttention(nn.Module):
v = v.view(bsz, seqlen1, self.n_heads, self.head_dim)
q, k = self.q_norm1(q), self.k_norm1(k)
- output = optimized_attention(q.permute(0, 2, 1, 3), k.permute(0, 2, 1, 3), v.permute(0, 2, 1, 3), self.n_heads, skip_reshape=True, transformer_options=transformer_options)
+ q, k, v = AttentionTensorContainer(q.permute(0, 2, 1, 3)), AttentionTensorContainer(k.permute(0, 2, 1, 3)), AttentionTensorContainer(v.permute(0, 2, 1, 3))
+ output = optimized_attention(q, k, v, self.n_heads, skip_reshape=True, preferred_attention=self.comfy_attention, transformer_options=transformer_options)
c = self.w1o(output)
return c
@@ -104,6 +106,7 @@ class SingleAttention(nn.Module):
class DoubleAttention(nn.Module):
def __init__(self, dim, n_heads, mh_qknorm=False, dtype=None, device=None, operations=None):
super().__init__()
+ self.comfy_attention = ComfyAttention()
self.n_heads = n_heads
self.head_dim = dim // n_heads
@@ -168,7 +171,9 @@ class DoubleAttention(nn.Module):
torch.cat([cv, xv], dim=1),
)
- output = optimized_attention(q.permute(0, 2, 1, 3), k.permute(0, 2, 1, 3), v.permute(0, 2, 1, 3), self.n_heads, skip_reshape=True, transformer_options=transformer_options)
+ del cq, ck, cv, xq, xk, xv
+ q, k, v = AttentionTensorContainer(q.permute(0, 2, 1, 3)), AttentionTensorContainer(k.permute(0, 2, 1, 3)), AttentionTensorContainer(v.permute(0, 2, 1, 3))
+ output = optimized_attention(q, k, v, self.n_heads, skip_reshape=True, preferred_attention=self.comfy_attention, transformer_options=transformer_options)
c, x = output.split([seqlen1, seqlen2], dim=1)
c = self.w1o(c)
diff --git a/comfy/ldm/boogu/model.py b/comfy/ldm/boogu/model.py
index ca88bdeb1..4dd4ff414 100644
--- a/comfy/ldm/boogu/model.py
+++ b/comfy/ldm/boogu/model.py
@@ -12,7 +12,7 @@ from einops import rearrange
import comfy.ldm.common_dit
import comfy.ldm.omnigen.omnigen2
-from comfy.ldm.modules.attention import optimized_attention_masked
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention_masked
from comfy.ldm.omnigen.omnigen2 import (
OmniGen2RotaryPosEmbed,
Lumina2CombinedTimestepCaptionEmbedding,
@@ -58,6 +58,7 @@ class BooguDoubleStreamProcessor(nn.Module):
query = torch.cat([instruct_q, img_q], dim=1)
key = torch.cat([instruct_k, img_k], dim=1)
value = torch.cat([instruct_v, img_v], dim=1)
+ del instruct_q, instruct_k, instruct_v, img_q, img_k, img_v
query = query.view(batch_size, -1, attn.heads, attn.dim_head)
key = key.view(batch_size, -1, attn.kv_heads, attn.dim_head)
@@ -70,12 +71,12 @@ class BooguDoubleStreamProcessor(nn.Module):
query = apply_rotary_emb(query, rotary_emb)
key = apply_rotary_emb(key, rotary_emb)
- query = query.transpose(1, 2)
- key = key.transpose(1, 2)
- value = value.transpose(1, 2)
+ query = AttentionTensorContainer(query.transpose(1, 2))
+ key = AttentionTensorContainer(key.transpose(1, 2))
+ value = AttentionTensorContainer(value.transpose(1, 2))
gqa_kwargs = {"enable_gqa": True} if attn.kv_heads < attn.heads else {}
- hidden_states = optimized_attention_masked(query, key, value, attn.heads, attention_mask, skip_reshape=True, transformer_options=transformer_options, **gqa_kwargs)
+ hidden_states = optimized_attention_masked(query, key, value, attn.heads, attention_mask, skip_reshape=True, preferred_attention=attn.comfy_attention, transformer_options=transformer_options, **gqa_kwargs)
# Split back to instruction/image, apply per-stream output projections, recombine.
instruct_hidden_states = self.instruct_out(hidden_states[:, :L_instruct])
@@ -90,6 +91,7 @@ class BooguJointAttention(nn.Module):
# Holds the shared q/k RMSNorm + final output projection
def __init__(self, dim, head_dim, heads, kv_heads, eps=1e-5, dtype=None, device=None, operations=None):
super().__init__()
+ self.comfy_attention = ComfyAttention()
self.heads = heads
self.kv_heads = kv_heads
self.dim_head = head_dim
diff --git a/comfy/ldm/ernie/model.py b/comfy/ldm/ernie/model.py
index 88a3775d0..678e4c99a 100644
--- a/comfy/ldm/ernie/model.py
+++ b/comfy/ldm/ernie/model.py
@@ -3,7 +3,7 @@ import torch
import torch.nn as nn
import torch.nn.functional as F
-from comfy.ldm.modules.attention import optimized_attention
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention
import comfy.model_management
import comfy.ops
import comfy.quant_ops
@@ -86,6 +86,7 @@ class TimestepEmbedding(nn.Module):
class ErnieImageAttention(nn.Module):
def __init__(self, query_dim: int, heads: int, dim_head: int, eps: float = 1e-6, operations=None, device=None, dtype=None):
super().__init__()
+ self.comfy_attention = ComfyAttention()
self.heads = heads
self.head_dim = dim_head
self.inner_dim = heads * dim_head
@@ -124,10 +125,12 @@ class ErnieImageAttention(nn.Module):
if image_rotary_emb is not None:
query, key = comfy.quant_ops.ck.apply_rope_split_half(query, key, image_rotary_emb)
- q_flat = query.reshape(B, S, -1)
- k_flat = key.reshape(B, S, -1)
+ q_flat = AttentionTensorContainer(query.reshape(B, S, -1))
+ k_flat = AttentionTensorContainer(key.reshape(B, S, -1))
+ v_flat = AttentionTensorContainer(v_flat)
+ del query, key
- hidden_states = optimized_attention(q_flat, k_flat, v_flat, self.heads, mask=attention_mask)
+ hidden_states = optimized_attention(q_flat, k_flat, v_flat, self.heads, mask=attention_mask, preferred_attention=self.comfy_attention)
return self.to_out[0](hidden_states)
diff --git a/comfy/ldm/hydit/attn_layers.py b/comfy/ldm/hydit/attn_layers.py
index 3ca25a5df..192cce591 100644
--- a/comfy/ldm/hydit/attn_layers.py
+++ b/comfy/ldm/hydit/attn_layers.py
@@ -1,7 +1,7 @@
import torch
import torch.nn as nn
from typing import Tuple, Union, Optional
-from comfy.ldm.modules.attention import optimized_attention
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention
def reshape_for_broadcast(freqs_cis: Union[torch.Tensor, Tuple[torch.Tensor]], x: torch.Tensor, head_first=False):
@@ -113,6 +113,7 @@ class CrossAttention(nn.Module):
factory_kwargs = {'device': device, 'dtype': dtype}
super().__init__()
self.attn_precision = attn_precision
+ self.comfy_attention = ComfyAttention()
self.qdim = qdim
self.kdim = kdim
self.num_heads = num_heads
@@ -156,12 +157,15 @@ class CrossAttention(nn.Module):
qq, _ = apply_rotary_emb(q, None, freqs_cis_img)
assert qq.shape == q.shape, f'qq: {qq.shape}, q: {q.shape}'
q = qq
+ del qq
q = q.transpose(-2, -3).contiguous() # q -> B, L1, H, C - B, H, L1, C
k = k.transpose(-2, -3).contiguous() # k -> B, L2, H, C - B, H, C, L2
v = v.transpose(-2, -3).contiguous()
- context = optimized_attention(q, k, v, self.num_heads, skip_reshape=True, attn_precision=self.attn_precision)
+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)
+ del kv
+ context = optimized_attention(q, k, v, self.num_heads, skip_reshape=True, attn_precision=self.attn_precision, preferred_attention=self.comfy_attention)
out = self.out_proj(context) # context.reshape - B, L1, -1
out = self.proj_drop(out)
@@ -178,6 +182,7 @@ class Attention(nn.Module):
def __init__(self, dim, num_heads, qkv_bias=True, qk_norm=False, attn_drop=0., proj_drop=0., attn_precision=None, dtype=None, device=None, operations=None):
super().__init__()
self.attn_precision = attn_precision
+ self.comfy_attention = ComfyAttention()
self.dim = dim
self.num_heads = num_heads
assert self.dim % num_heads == 0, 'dim should be divisible by num_heads'
@@ -208,8 +213,11 @@ class Attention(nn.Module):
assert qq.shape == q.shape and kk.shape == k.shape, \
f'qq: {qq.shape}, q: {q.shape}, kk: {kk.shape}, k: {k.shape}'
q, k = qq, kk
+ del qq, kk
- x = optimized_attention(q, k, v, self.num_heads, skip_reshape=True, attn_precision=self.attn_precision)
+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)
+ del qkv
+ x = optimized_attention(q, k, v, self.num_heads, skip_reshape=True, attn_precision=self.attn_precision, preferred_attention=self.comfy_attention)
x = self.out_proj(x)
x = self.proj_drop(x)
diff --git a/comfy/ldm/hydit/poolers.py b/comfy/ldm/hydit/poolers.py
index c1b878ed6..db685ac23 100644
--- a/comfy/ldm/hydit/poolers.py
+++ b/comfy/ldm/hydit/poolers.py
@@ -1,11 +1,12 @@
import torch
import torch.nn as nn
-from comfy.ldm.modules.attention import optimized_attention
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention
import comfy.ops
class AttentionPool(nn.Module):
def __init__(self, spacial_dim: int, embed_dim: int, num_heads: int, output_dim: int = None, dtype=None, device=None, operations=None):
super().__init__()
+ self.comfy_attention = ComfyAttention()
self.positional_embedding = nn.Parameter(torch.empty(spacial_dim + 1, embed_dim, dtype=dtype, device=device))
self.k_proj = operations.Linear(embed_dim, embed_dim, dtype=dtype, device=device)
self.q_proj = operations.Linear(embed_dim, embed_dim, dtype=dtype, device=device)
@@ -30,7 +31,8 @@ class AttentionPool(nn.Module):
k = k.view(k.shape[0], batch_size * self.num_heads, head_dim).transpose(0, 1).view(batch_size, self.num_heads, -1, head_dim)
v = v.view(v.shape[0], batch_size * self.num_heads, head_dim).transpose(0, 1).view(batch_size, self.num_heads, -1, head_dim)
- attn_output = optimized_attention(q, k, v, self.num_heads, skip_reshape=True).transpose(0, 1)
+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)
+ attn_output = optimized_attention(q, k, v, self.num_heads, skip_reshape=True, preferred_attention=self.comfy_attention).transpose(0, 1)
attn_output = self.c_proj(attn_output)
return attn_output.squeeze(0)
diff --git a/comfy/ldm/lens/model.py b/comfy/ldm/lens/model.py
index cd5015ddc..07b9194a6 100644
--- a/comfy/ldm/lens/model.py
+++ b/comfy/ldm/lens/model.py
@@ -12,7 +12,7 @@ import comfy.ldm.flux.layers
import comfy.patcher_extension
from comfy.ldm.flux.layers import EmbedND
from comfy.ldm.flux.math import apply_rope
-from comfy.ldm.modules.attention import optimized_attention
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention
def _lens_time_proj(t: torch.Tensor, dim: int = 256) -> torch.Tensor:
@@ -105,6 +105,7 @@ class LensJointAttention(nn.Module):
operations=None,
) -> None:
super().__init__()
+ self.comfy_attention = ComfyAttention()
self.inner_dim = out_dim if out_dim is not None else dim_head * heads
self.heads = self.inner_dim // dim_head
self.dim_head = dim_head
@@ -148,6 +149,7 @@ class LensJointAttention(nn.Module):
txt_q, txt_k, txt_v = txt_qkv.unbind(dim=2)
txt_q = self.norm_added_q(txt_q)
txt_k = self.norm_added_k(txt_k)
+ del txt_qkv
# [B, S, H, D] → [B, H, S, D] for attention, dels to avoid VRAM peaks
q = torch.cat([img_q, txt_q], dim=1).transpose(1, 2)
@@ -167,9 +169,11 @@ class LensJointAttention(nn.Module):
)
attention_mask = attention_mask.to(q.dtype)
+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)
out = optimized_attention(
q, k, v, self.heads, mask=attention_mask, skip_reshape=True,
transformer_options=transformer_options,
+ preferred_attention=self.comfy_attention,
)
img_out = self.to_out[1](self.to_out[0](out[:, :seq_img, :]))
diff --git a/comfy/ldm/minimax_music/dit.py b/comfy/ldm/minimax_music/dit.py
index 211e0d7db..258e19b85 100644
--- a/comfy/ldm/minimax_music/dit.py
+++ b/comfy/ldm/minimax_music/dit.py
@@ -6,7 +6,7 @@ from torch import nn
import comfy.model_management
import comfy.ops
import comfy.quant_ops
-from comfy.ldm.modules.attention import optimized_attention_for_device
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention_for_device
MAX_CONDITION_FRAMES = 200
@@ -66,6 +66,7 @@ def _apply_rope(x, rotation_matrix):
class Attention(nn.Module):
def __init__(self, dim, dim_heads, dtype, device, operations):
super().__init__()
+ self.comfy_attention = ComfyAttention()
self.num_heads = dim // dim_heads
self.dim_heads = dim_heads
self.to_qkv = operations.Linear(dim, dim * 3, bias=False, dtype=dtype, device=device)
@@ -85,8 +86,10 @@ class Attention(nn.Module):
rotated_q, rotated_k = comfy.quant_ops.ck.apply_rope_split_half(q[..., :rotary_dims], k[..., :rotary_dims], rotation_matrix)
q = torch.cat((rotated_q, q[..., rotary_dims:]), dim=-1)
k = torch.cat((rotated_k, k[..., rotary_dims:]), dim=-1)
+ del rotated_q, rotated_k
attention = optimized_attention_for_device(q.device)
- out = attention(q, k, v, self.num_heads, skip_reshape=True)
+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)
+ out = attention(q, k, v, self.num_heads, skip_reshape=True, preferred_attention=self.comfy_attention)
return self.to_out(out)
diff --git a/comfy/ldm/omnigen/omnigen2.py b/comfy/ldm/omnigen/omnigen2.py
index d18a9f6d0..b265edd16 100644
--- a/comfy/ldm/omnigen/omnigen2.py
+++ b/comfy/ldm/omnigen/omnigen2.py
@@ -9,7 +9,7 @@ from einops import rearrange, repeat
from comfy.ldm.lightricks.model import Timesteps
from comfy.ldm.flux.layers import EmbedND
from comfy.ldm.flux.math import apply_rope1
-from comfy.ldm.modules.attention import optimized_attention_masked
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention_masked
import comfy.model_management
import comfy.ldm.common_dit
@@ -102,6 +102,7 @@ class Lumina2CombinedTimestepCaptionEmbedding(nn.Module):
class Attention(nn.Module):
def __init__(self, query_dim: int, dim_head: int, heads: int, kv_heads: int, eps: float = 1e-5, bias: bool = False, dtype=None, device=None, operations=None):
super().__init__()
+ self.comfy_attention = ComfyAttention()
self.heads = heads
self.kv_heads = kv_heads
self.dim_head = dim_head
@@ -137,12 +138,12 @@ class Attention(nn.Module):
query = apply_rotary_emb(query, image_rotary_emb)
key = apply_rotary_emb(key, image_rotary_emb)
- query = query.transpose(1, 2)
- key = key.transpose(1, 2)
- value = value.transpose(1, 2)
+ query = AttentionTensorContainer(query.transpose(1, 2))
+ key = AttentionTensorContainer(key.transpose(1, 2))
+ value = AttentionTensorContainer(value.transpose(1, 2))
gqa_kwargs = {"enable_gqa": True} if self.kv_heads < self.heads else {}
- hidden_states = optimized_attention_masked(query, key, value, self.heads, attention_mask, skip_reshape=True, transformer_options=transformer_options, **gqa_kwargs)
+ hidden_states = optimized_attention_masked(query, key, value, self.heads, attention_mask, skip_reshape=True, preferred_attention=self.comfy_attention, transformer_options=transformer_options, **gqa_kwargs)
hidden_states = self.to_out[0](hidden_states)
return hidden_states
diff --git a/comfy/ldm/pixart/blocks.py b/comfy/ldm/pixart/blocks.py
index 2225076e5..b27a66186 100644
--- a/comfy/ldm/pixart/blocks.py
+++ b/comfy/ldm/pixart/blocks.py
@@ -7,7 +7,7 @@ import torch.nn.functional as F
from einops import rearrange
from comfy.ldm.modules.diffusionmodules.mmdit import TimestepEmbedder, Mlp, timestep_embedding
-from comfy.ldm.modules.attention import optimized_attention
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention
# if model_management.xformers_enabled():
# import xformers.ops
@@ -25,6 +25,7 @@ def t2i_modulate(x, shift, scale):
class MultiHeadCrossAttention(nn.Module):
def __init__(self, d_model, num_heads, attn_drop=0., proj_drop=0., dtype=None, device=None, operations=None, **kwargs):
super(MultiHeadCrossAttention, self).__init__()
+ self.comfy_attention = ComfyAttention()
assert d_model % num_heads == 0, "d_model must be divisible by num_heads"
self.d_model = d_model
@@ -71,7 +72,9 @@ class MultiHeadCrossAttention(nn.Module):
# attn_mask = torch.block_diag(attn_mask, attn_mask_template)
# x = optimized_attention(q, k, v, self.num_heads, mask=attn_mask, skip_reshape=True)
- x = optimized_attention(q.view(B, -1, C), k.view(B, -1, C), v.view(B, -1, C), self.num_heads, mask=None)
+ q, k, v = AttentionTensorContainer(q.view(B, -1, C)), AttentionTensorContainer(k.view(B, -1, C)), AttentionTensorContainer(v.view(B, -1, C))
+ del kv
+ x = optimized_attention(q, k, v, self.num_heads, mask=None, preferred_attention=self.comfy_attention)
x = self.proj(x)
x = self.proj_drop(x)
return x
@@ -87,6 +90,7 @@ class AttentionKVCompress(nn.Module):
qkv_bias (bool: If True, add a learnable bias to query, key, value.
"""
super().__init__()
+ self.comfy_attention = ComfyAttention()
assert dim % num_heads == 0, 'dim should be divisible by num_heads'
self.num_heads = num_heads
self.head_dim = dim // num_heads
@@ -169,7 +173,9 @@ class AttentionKVCompress(nn.Module):
# attention 2
q, k, v = map(lambda t: t.transpose(1, 2), (q, k, v),)
- x = optimized_attention(q, k, v, self.num_heads, mask=None, skip_reshape=True)
+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)
+ del qkv
+ x = optimized_attention(q, k, v, self.num_heads, mask=None, skip_reshape=True, preferred_attention=self.comfy_attention)
x = x.view(B, N, C)
x = self.proj(x)
diff --git a/comfy/ldm/pixeldit/model.py b/comfy/ldm/pixeldit/model.py
index 3b30b9226..7f9758282 100644
--- a/comfy/ldm/pixeldit/model.py
+++ b/comfy/ldm/pixeldit/model.py
@@ -6,7 +6,7 @@ import comfy.ldm.common_dit
import comfy.patcher_extension
from comfy.ldm.flux.math import apply_rope, rope
from comfy.ldm.hidream.model import FeedForwardSwiGLU
-from comfy.ldm.modules.attention import optimized_attention
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention
from comfy.ldm.modules.diffusionmodules.mmdit import TimestepEmbedder
from .modules import (
@@ -27,6 +27,7 @@ class MMDiTJointAttention(nn.Module):
"""
def __init__(self, dim, num_heads=8, qkv_bias=False, dtype=None, device=None, operations=None):
super().__init__()
+ self.comfy_attention = ComfyAttention()
assert dim % num_heads == 0
self.num_heads = num_heads
self.head_dim = dim // num_heads
@@ -66,10 +67,12 @@ class MMDiTJointAttention(nn.Module):
k_joint = torch.cat([ky, kx], dim=2)
v_joint = torch.cat([vy, vx], dim=2)
+ del qkv_x, qkv_y, qx, kx, vx, qy, ky, vy
+ q_joint, k_joint, v_joint = AttentionTensorContainer(q_joint), AttentionTensorContainer(k_joint), AttentionTensorContainer(v_joint)
out_joint = optimized_attention(
q_joint, k_joint, v_joint, H,
mask=attn_mask, skip_reshape=True, skip_output_reshape=True,
- transformer_options=transformer_options,
+ transformer_options=transformer_options, preferred_attention=self.comfy_attention,
)
out_y = out_joint[:, :, :Ny, :].transpose(1, 2).reshape(B, Ny, H * D)
diff --git a/comfy/ldm/pixeldit/modules.py b/comfy/ldm/pixeldit/modules.py
index 4b1e538c7..88426e5a0 100644
--- a/comfy/ldm/pixeldit/modules.py
+++ b/comfy/ldm/pixeldit/modules.py
@@ -2,7 +2,7 @@ import torch
import torch.nn as nn
from comfy.ldm.flux.math import apply_rope, rope
-from comfy.ldm.modules.attention import optimized_attention
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention
from comfy.ldm.modules.diffusionmodules.mmdit import Mlp, get_1d_sincos_pos_embed_from_grid_torch
@@ -58,6 +58,7 @@ class RotaryAttention(nn.Module):
"""Single-stream self-attention with rotary positional encoding (used inside PiTBlock)."""
def __init__(self, dim, num_heads=8, qkv_bias=False, dtype=None, device=None, operations=None):
super().__init__()
+ self.comfy_attention = ComfyAttention()
assert dim % num_heads == 0
self.num_heads = num_heads
self.head_dim = dim // num_heads
@@ -73,7 +74,9 @@ class RotaryAttention(nn.Module):
qkv = self.qkv(x).reshape(B, N, 3, H, D).permute(2, 0, 3, 1, 4)
q, k, v = qkv.unbind(0)
q, k = apply_rope(self.q_norm(q), self.k_norm(k), pos[None, None])
- x = optimized_attention(q, k, v, H, mask=mask, skip_reshape=True, transformer_options=transformer_options)
+ del qkv
+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)
+ x = optimized_attention(q, k, v, H, mask=mask, skip_reshape=True, preferred_attention=self.comfy_attention, transformer_options=transformer_options)
return self.proj(x)
diff --git a/comfy/ldm/qwen_image/model.py b/comfy/ldm/qwen_image/model.py
index e49886dd9..b06a8fe0e 100644
--- a/comfy/ldm/qwen_image/model.py
+++ b/comfy/ldm/qwen_image/model.py
@@ -6,7 +6,7 @@ from typing import Optional, Tuple
from einops import repeat
from comfy.ldm.lightricks.model import TimestepEmbedding, Timesteps
-from comfy.ldm.modules.attention import optimized_attention_masked
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention_masked
from comfy.ldm.flux.layers import EmbedND
import comfy.ldm.common_dit
import comfy.patcher_extension
@@ -108,6 +108,7 @@ class Attention(nn.Module):
operations=None
):
super().__init__()
+ self.comfy_attention = ComfyAttention()
self.inner_dim = out_dim if out_dim is not None else dim_head * heads
self.inner_kv_dim = self.inner_dim
self.heads = heads
@@ -172,6 +173,7 @@ class Attention(nn.Module):
joint_query = torch.cat([txt_query, img_query], dim=2)
joint_key = torch.cat([txt_key, img_key], dim=2)
joint_value = torch.cat([txt_value, img_value], dim=2)
+ del img_query, img_key, img_value, txt_query, txt_key, txt_value
if encoder_hidden_states_mask is not None:
attn_mask = torch.zeros((batch_size, 1, seq_txt + seq_img), dtype=hidden_states.dtype, device=hidden_states.device)
@@ -179,7 +181,7 @@ class Attention(nn.Module):
else:
attn_mask = None
- extra_options["img_slice"] = [txt_query.shape[2], joint_query.shape[2]]
+ extra_options["img_slice"] = [seq_txt, joint_query.shape[2]]
if "attn1_patch" in transformer_patches:
patch = transformer_patches["attn1_patch"]
for p in patch:
@@ -188,10 +190,13 @@ class Attention(nn.Module):
joint_query = apply_rope1(joint_query, image_rotary_emb)
joint_key = apply_rope1(joint_key, image_rotary_emb)
+ joint_query = AttentionTensorContainer(joint_query)
+ joint_key = AttentionTensorContainer(joint_key)
+ joint_value = AttentionTensorContainer(joint_value)
joint_hidden_states = optimized_attention_masked(joint_query, joint_key, joint_value, self.heads,
attn_mask, transformer_options=transformer_options,
- skip_reshape=True)
+ skip_reshape=True, preferred_attention=self.comfy_attention)
txt_attn_output = joint_hidden_states[:, :seq_txt, :]
img_attn_output = joint_hidden_states[:, seq_txt:, :]
diff --git a/comfy/ldm/qwen_image21/model.py b/comfy/ldm/qwen_image21/model.py
index b804b9eca..e50d6c3cb 100644
--- a/comfy/ldm/qwen_image21/model.py
+++ b/comfy/ldm/qwen_image21/model.py
@@ -12,7 +12,7 @@ import comfy.rmsnorm
from comfy.ldm.flux.layers import EmbedND, timestep_embedding
from comfy.ldm.flux.math import apply_rope1
from comfy.ldm.lightricks.model import TimestepEmbedding
-from comfy.ldm.modules.attention import ComfyAttention, optimized_attention
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention
from comfy.ldm.wan.model_animate2 import PoseBranchCache
@@ -104,6 +104,7 @@ class Attention(nn.Module):
q, k = comfy.quant_ops.ck.rms_rope(q, k, pe, q_scale, k_scale, self.norm_q.eps)
comfy.ops.uncast_bias_weight(self.norm_q, q_scale, None, q_stream)
comfy.ops.uncast_bias_weight(self.norm_k, k_scale, None, k_stream)
+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)
return self.to_out[0](attn_fn(q, k, v, self.heads, preferred_attention=self.comfy_attention))
@@ -196,11 +197,18 @@ class QwenImage21FunControl(nn.Module):
def block_causal_attention(segments, transformer_options={}, cache=None, block_index=0, prefix_len=0):
# segments: (start, end, mask); text segments get a causal mask, image blocks attend to everything before their end
def attn(q, k, v, heads, preferred_attention=None):
+ q, k, v = q.take(), k.take(), v.take()
if cache is not None:
# K and V stacked on dim 1 so batch stays first and quantized rows are per token and head
cache.put(block_index, torch.stack([k[:, :prefix_len], v[:, :prefix_len]], dim=1))
- outs = [optimized_attention(q[:, start:end].flatten(2), k[:, :end].flatten(2), v[:, :end].flatten(2), heads, mask=mask, transformer_options=transformer_options, preferred_attention=preferred_attention)
- for start, end, mask in segments]
+ outs = []
+ for i, (start, end, mask) in enumerate(segments):
+ query = AttentionTensorContainer(q[:, start:end].flatten(2))
+ key = AttentionTensorContainer(k[:, :end].flatten(2))
+ value = AttentionTensorContainer(v[:, :end].flatten(2))
+ if i == len(segments) - 1:
+ del q, k, v
+ outs.append(optimized_attention(query, key, value, heads, mask=mask, transformer_options=transformer_options, preferred_attention=preferred_attention))
return torch.cat(outs, dim=1) if len(outs) > 1 else outs[0]
return attn
@@ -208,7 +216,10 @@ def block_causal_attention(segments, transformer_options={}, cache=None, block_i
def prefix_cached_attention(prefix_k, prefix_v, transformer_options={}):
# target-only queries: block-causal reduces to full attention over [cached prefix, target]
def attn(q, k, v, heads, preferred_attention=None):
- return optimized_attention(q.flatten(2), torch.cat([prefix_k, k], dim=1).flatten(2), torch.cat([prefix_v, v], dim=1).flatten(2), heads, transformer_options=transformer_options, preferred_attention=preferred_attention)
+ q = AttentionTensorContainer(q.take().flatten(2))
+ k = AttentionTensorContainer(torch.cat([prefix_k, k.take()], dim=1).flatten(2))
+ v = AttentionTensorContainer(torch.cat([prefix_v, v.take()], dim=1).flatten(2))
+ return optimized_attention(q, k, v, heads, transformer_options=transformer_options, preferred_attention=preferred_attention)
return attn