Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Add comfy_attention and AttentionTensorContainer support to rest of models (#16806)

comfyanonymousdddb41aad74e
diff --git a/comfy/ldm/hidream/model.py b/comfy/ldm/hidream/model.py
index 28d81c79e..85f278197 100644
--- a/comfy/ldm/hidream/model.py
+++ b/comfy/ldm/hidream/model.py
@@ -11,7 +11,7 @@ import torch.nn.functional as F
from comfy.ldm.flux.math import apply_rope, rope
from comfy.ldm.flux.layers import LastLayer
-from comfy.ldm.modules.attention import optimized_attention
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention
import comfy.model_management
import comfy.patcher_extension
import comfy.ldm.common_dit
@@ -72,8 +72,12 @@ class TimestepEmbed(nn.Module):
return t_emb
-def attention(query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, transformer_options={}):
- return optimized_attention(query.view(query.shape[0], -1, query.shape[-1] * query.shape[-2]), key.view(key.shape[0], -1, key.shape[-1] * key.shape[-2]), value.view(value.shape[0], -1, value.shape[-1] * value.shape[-2]), query.shape[2], transformer_options=transformer_options)
+def attention(query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, transformer_options={}, preferred_attention=None):
+ if isinstance(query, AttentionTensorContainer):
+ query, key, value = query.take(), key.take(), value.take()
+ heads = query.shape[2]
+ query, key, value = AttentionTensorContainer(query.flatten(2)), AttentionTensorContainer(key.flatten(2)), AttentionTensorContainer(value.flatten(2))
+ return optimized_attention(query, key, value, heads, preferred_attention=preferred_attention, transformer_options=transformer_options)
class HiDreamAttnProcessor_flashattn:
@@ -120,11 +124,13 @@ class HiDreamAttnProcessor_flashattn:
query = torch.cat([query_i, query_t], dim=1)
key = torch.cat([key_i, key_t], dim=1)
value = torch.cat([value_i, value_t], dim=1)
+ del query_t, key_t, value_t
else:
query = query_i
key = key_i
value = value_i
+ del query_i, key_i, value_i
if query.shape[-1] == rope.shape[-3] * 2:
query, key = apply_rope(query, key, rope)
else:
@@ -133,8 +139,10 @@ class HiDreamAttnProcessor_flashattn:
query_1, key_1 = apply_rope(query_1, key_1, rope)
query = torch.cat([query_1, query_2], dim=-1)
key = torch.cat([key_1, key_2], dim=-1)
+ del query_1, query_2, key_1, key_2
- hidden_states = attention(query, key, value, transformer_options=transformer_options)
+ query, key, value = AttentionTensorContainer(query), AttentionTensorContainer(key), AttentionTensorContainer(value)
+ hidden_states = attention(query, key, value, transformer_options=transformer_options, preferred_attention=attn.comfy_attention)
if not attn.single:
hidden_states_i, hidden_states_t = torch.split(hidden_states, [num_image_tokens, num_text_tokens], dim=1)
@@ -162,6 +170,7 @@ class HiDreamAttention(nn.Module):
):
# super(Attention, self).__init__()
super().__init__()
+ self.comfy_attention = ComfyAttention()
self.inner_dim = out_dim if out_dim is not None else dim_head * heads
self.query_dim = query_dim
self.upcast_attention = upcast_attention
diff --git a/comfy/ldm/hidream_o1/attention.py b/comfy/ldm/hidream_o1/attention.py
index afb2be9b8..b857424c7 100644
--- a/comfy/ldm/hidream_o1/attention.py
+++ b/comfy/ldm/hidream_o1/attention.py
@@ -7,10 +7,10 @@ gen half hit the user's preferred backend via optimized_attention.
import torch
import comfy.ops
-from comfy.ldm.modules.attention import optimized_attention
+from comfy.ldm.modules.attention import AttentionTensorContainer, optimized_attention
-def make_two_pass_attention(ar_len: int, transformer_options=None):
+def make_two_pass_attention(ar_len: int, transformer_options=None, preferred_attention=None):
"""Build a two-pass attention callable. AR pass uses SDPA-causal directly, gen pass routes through optimized_attention.
The AR pass goes through SDPA directand bypasses wrappers, it is only ~1% of T at typical edit sizes.
"""
@@ -19,20 +19,23 @@ def make_two_pass_attention(ar_len: int, transformer_options=None):
B, H, T, D = q.shape
if T < k.shape[2]: # KV-cache hot path: Q is shorter than K/V (cached AR prefix is in K/V only), all fresh Q positions are in the gen region, single full-attention call
- out = optimized_attention(q, k, v, heads, mask=None, skip_reshape=True, skip_output_reshape=True, transformer_options=transformer_options, enable_gqa=enable_gqa)
+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)
+ out = optimized_attention(q, k, v, heads, mask=None, skip_reshape=True, skip_output_reshape=True, transformer_options=transformer_options, preferred_attention=preferred_attention, enable_gqa=enable_gqa)
elif ar_len >= T:
out = comfy.ops.scaled_dot_product_attention(q, k, v, attn_mask=None, dropout_p=0.0, is_causal=True, enable_gqa=enable_gqa)
elif ar_len <= 0:
- out = optimized_attention(q, k, v, heads, mask=None, skip_reshape=True, skip_output_reshape=True, transformer_options=transformer_options, enable_gqa=enable_gqa)
+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)
+ out = optimized_attention(q, k, v, heads, mask=None, skip_reshape=True, skip_output_reshape=True, transformer_options=transformer_options, preferred_attention=preferred_attention, enable_gqa=enable_gqa)
else:
out_ar = comfy.ops.scaled_dot_product_attention(
q[:, :, :ar_len], k[:, :, :ar_len], v[:, :, :ar_len],
attn_mask=None, dropout_p=0.0, is_causal=True, enable_gqa=enable_gqa,
)
+ q, k, v = AttentionTensorContainer(q[:, :, ar_len:]), AttentionTensorContainer(k), AttentionTensorContainer(v)
out_gen = optimized_attention(
- q[:, :, ar_len:], k, v, heads,
+ q, k, v, heads,
mask=None, skip_reshape=True, skip_output_reshape=True,
- transformer_options=transformer_options, enable_gqa=enable_gqa,
+ transformer_options=transformer_options, preferred_attention=preferred_attention, enable_gqa=enable_gqa,
)
out = torch.cat([out_ar, out_gen], dim=2)
diff --git a/comfy/ldm/hidream_o1/model.py b/comfy/ldm/hidream_o1/model.py
index a223e706f..03d2a73d0 100644
--- a/comfy/ldm/hidream_o1/model.py
+++ b/comfy/ldm/hidream_o1/model.py
@@ -15,6 +15,7 @@ import torch
import torch.nn as nn
import comfy.patcher_extension
+from comfy.ldm.modules.attention import ComfyAttention
from comfy.ldm.modules.diffusionmodules.mmdit import TimestepEmbedder
from comfy.text_encoders.llama import Llama2_
from comfy.text_encoders.qwen35 import Qwen35VisionModel
@@ -113,6 +114,8 @@ class HiDreamO1Transformer(nn.Module):
self.visual = Qwen35VisionModel(vision_cfg, device=device, dtype=dtype, ops=operations)
self.language_model = Llama2_(text_cfg, device=device, dtype=dtype, ops=operations)
+ for layer in self.language_model.layers:
+ layer.self_attn.comfy_attention = ComfyAttention()
self.t_embedder1 = TimestepEmbedder(
text_cfg.hidden_size, device=device, dtype=dtype, operations=operations,
)
@@ -197,7 +200,6 @@ class HiDreamO1Transformer(nn.Module):
freqs_cis = self.language_model.compute_freqs_cis(position_ids[0].to(x.device), x.device)
freqs_cis = tuple(t.to(x.dtype) for t in freqs_cis)
- two_pass_attn = make_two_pass_attention(ar_len, transformer_options=transformer_options)
patches_replace = transformer_options.get("patches_replace", {})
blocks_replace = patches_replace.get("dit", {})
transformer_options["total_blocks"] = len(self.language_model.layers)
@@ -231,6 +233,7 @@ class HiDreamO1Transformer(nn.Module):
hidden_states = inputs_embeds[:, cache_len:]
sliced_freqs = tuple(t[..., cache_len:, :] for t in freqs_cis)
for i, layer in enumerate(self.language_model.layers):
+ two_pass_attn = make_two_pass_attention(ar_len, transformer_options=transformer_options, preferred_attention=layer.self_attn.comfy_attention)
transformer_options["block_index"] = i
K_i, V_i = kv_cache["kv"][i]
hidden_states, _ = layer(
@@ -243,6 +246,7 @@ class HiDreamO1Transformer(nn.Module):
past_kv_cold = () if can_cache else None
hidden_states = inputs_embeds
for i, layer in enumerate(self.language_model.layers):
+ two_pass_attn = make_two_pass_attention(ar_len, transformer_options=transformer_options, preferred_attention=layer.self_attn.comfy_attention)
transformer_options["block_index"] = i
if ("double_block", i) in blocks_replace:
def block_wrap(args, _layer=layer):
diff --git a/comfy/ldm/hunyuan3dv2_1/hunyuandit.py b/comfy/ldm/hunyuan3dv2_1/hunyuandit.py
index 4e4819fe3..32ca98503 100644
--- a/comfy/ldm/hunyuan3dv2_1/hunyuandit.py
+++ b/comfy/ldm/hunyuan3dv2_1/hunyuandit.py
@@ -2,7 +2,7 @@ import math
import torch
import torch.nn as nn
import torch.nn.functional as F
-from comfy.ldm.modules.attention import optimized_attention
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention
import comfy.model_management
class GELU(nn.Module):
@@ -288,6 +288,7 @@ class CrossAttention(nn.Module):
**kwargs,
):
super().__init__()
+ self.comfy_attention = ComfyAttention()
self.qdim = qdim
self.kdim = kdim
@@ -338,12 +339,14 @@ class CrossAttention(nn.Module):
q = self.q_norm(q)
k = self.k_norm(k)
+ q, k, v = AttentionTensorContainer(q.reshape(b, s1, -1)), AttentionTensorContainer(k.reshape(b, s2, -1)), AttentionTensorContainer(v)
+ del kv
x = optimized_attention(
- q.reshape(b, s1, self.num_heads * self.head_dim),
- k.reshape(b, s2, self.num_heads * self.head_dim),
+ q,
+ k,
v,
heads=self.num_heads,
- low_precision_attention=False,
+ low_precision_attention=False, preferred_attention=self.comfy_attention,
)
out = self.out_proj(x)
@@ -365,6 +368,7 @@ class Attention(nn.Module):
dtype = None
):
super().__init__()
+ self.comfy_attention = ComfyAttention()
self.dim = dim
self.num_heads = num_heads
self.head_dim = self.dim // num_heads
@@ -408,12 +412,14 @@ class Attention(nn.Module):
query = self.q_norm(query)
key = self.k_norm(key)
+ query, key, value = AttentionTensorContainer(query.reshape(B, N, -1)), AttentionTensorContainer(key.reshape(B, N, -1)), AttentionTensorContainer(value)
+ del qkv, qkv_combined
x = optimized_attention(
- query.reshape(B, N, self.num_heads * self.head_dim),
- key.reshape(B, N, self.num_heads * self.head_dim),
+ query,
+ key,
value,
heads=self.num_heads,
- low_precision_attention=False,
+ low_precision_attention=False, preferred_attention=self.comfy_attention,
)
x = self.out_proj(x)
diff --git a/comfy/ldm/lightricks/model.py b/comfy/ldm/lightricks/model.py
index dcbfa43ad..b011f5d81 100644
--- a/comfy/ldm/lightricks/model.py
+++ b/comfy/ldm/lightricks/model.py
@@ -11,6 +11,7 @@ import torch
from torch import nn
import comfy.patcher_extension
import comfy.ldm.modules.attention
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention
import comfy.ldm.common_dit
import comfy.model_management
import comfy.ops
@@ -393,7 +394,7 @@ class GuideAttentionMask:
self.tracked_mask[:, :, :, :guide_start] = log_w.view(1, 1, -1, 1)
-def _attention_with_guide_mask(q, k, v, heads, guide_mask, attn_precision, transformer_options):
+def _attention_with_guide_mask(q, k, v, heads, guide_mask, attn_precision, transformer_options, preferred_attention=None):
"""Apply the guide mask by partitioning Q into noisy and tracked-guide
groups, so each group needs only its own sub-mask. Avoids materializing
the (1,1,T,T) dense mask.
@@ -405,19 +406,19 @@ def _attention_with_guide_mask(q, k, v, heads, guide_mask, attn_precision, trans
if guide_start > 0: # In practice currently guides are always after noise, guard for safety if this changes.
out[:, :guide_start, :] = comfy.ldm.modules.attention.optimized_attention(
- q[:, :guide_start, :], k, v, heads, mask=guide_mask.noisy_mask,
- attn_precision=attn_precision, transformer_options=transformer_options,
+ AttentionTensorContainer(q[:, :guide_start, :]), AttentionTensorContainer(k), AttentionTensorContainer(v), heads, mask=guide_mask.noisy_mask,
+ attn_precision=attn_precision, transformer_options=transformer_options, preferred_attention=preferred_attention,
low_precision_attention=False, # sageattn mask support is unreliable
)
out[:, guide_start:tracked_end, :] = comfy.ldm.modules.attention.optimized_attention(
- q[:, guide_start:tracked_end, :], k, v, heads, mask=guide_mask.tracked_mask,
- attn_precision=attn_precision, transformer_options=transformer_options,
+ AttentionTensorContainer(q[:, guide_start:tracked_end, :]), AttentionTensorContainer(k), AttentionTensorContainer(v), heads, mask=guide_mask.tracked_mask,
+ attn_precision=attn_precision, transformer_options=transformer_options, preferred_attention=preferred_attention,
low_precision_attention=False,
)
if tracked_end < q.shape[1]: # Every guide token is tracked, and nothing comes after them, guard for safety if this changes.
out[:, tracked_end:, :] = comfy.ldm.modules.attention.optimized_attention(
- q[:, tracked_end:, :], k, v, heads,
- attn_precision=attn_precision, transformer_options=transformer_options,
+ AttentionTensorContainer(q[:, tracked_end:, :]), AttentionTensorContainer(k), AttentionTensorContainer(v), heads,
+ attn_precision=attn_precision, transformer_options=transformer_options, preferred_attention=preferred_attention,
)
return out
@@ -437,6 +438,7 @@ class CrossAttention(nn.Module):
operations=None,
):
super().__init__()
+ self.comfy_attention = ComfyAttention()
inner_dim = dim_head * heads
context_dim = query_dim if context_dim is None else context_dim
self.attn_precision = attn_precision
@@ -484,12 +486,11 @@ class CrossAttention(nn.Module):
q = apply_rotary_emb(q, pe)
k = apply_rotary_emb(k, pe if k_pe is None else k_pe)
- if mask is None:
- out = comfy.ldm.modules.attention.optimized_attention(q, k, v, self.heads, attn_precision=self.attn_precision, transformer_options=transformer_options)
- elif isinstance(mask, GuideAttentionMask):
- out = _attention_with_guide_mask(q, k, v, self.heads, mask, attn_precision=self.attn_precision, transformer_options=transformer_options)
+ if isinstance(mask, GuideAttentionMask):
+ out = _attention_with_guide_mask(q, k, v, self.heads, mask, attn_precision=self.attn_precision, transformer_options=transformer_options, preferred_attention=self.comfy_attention)
else:
- out = comfy.ldm.modules.attention.optimized_attention(q, k, v, self.heads, mask=mask, attn_precision=self.attn_precision, transformer_options=transformer_options)
+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)
+ out = comfy.ldm.modules.attention.optimized_attention(q, k, v, self.heads, mask=mask, attn_precision=self.attn_precision, transformer_options=transformer_options, preferred_attention=self.comfy_attention)
# Apply per-head gating if enabled
if self.to_gate_logits is not None:
diff --git a/comfy/ldm/modules/diffusionmodules/mmdit.py b/comfy/ldm/modules/diffusionmodules/mmdit.py
index 9ab3c463c..99407233b 100644
--- a/comfy/ldm/modules/diffusionmodules/mmdit.py
+++ b/comfy/ldm/modules/diffusionmodules/mmdit.py
@@ -4,7 +4,7 @@ from typing import Dict, Optional, List
import numpy as np
import torch
import torch.nn as nn
-from ..attention import optimized_attention
+from ..attention import AttentionTensorContainer, ComfyAttention, optimized_attention
from einops import rearrange, repeat
from .util import timestep_embedding
import comfy.ops
@@ -276,6 +276,7 @@ class SelfAttention(nn.Module):
operations=None,
):
super().__init__()
+ self.comfy_attention = ComfyAttention()
self.num_heads = num_heads
self.head_dim = dim // num_heads
@@ -315,8 +316,9 @@ class SelfAttention(nn.Module):
def forward(self, x: torch.Tensor) -> torch.Tensor:
q, k, v = self.pre_attention(x)
+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)
x = optimized_attention(
- q, k, v, heads=self.num_heads
+ q, k, v, heads=self.num_heads, preferred_attention=self.comfy_attention
)
x = self.post_attention(x)
return x
@@ -577,20 +579,23 @@ class DismantledBlock(nn.Module):
assert not self.pre_only
if self.x_block_self_attn:
qkv, qkv2, intermediates = self.pre_attention_x(x, c)
- attn, _ = optimized_attention(
+ qkv = tuple(AttentionTensorContainer(t) for t in qkv)
+ qkv2 = tuple(AttentionTensorContainer(t) for t in qkv2)
+ attn = optimized_attention(
qkv[0], qkv[1], qkv[2],
- num_heads=self.attn.num_heads,
+ heads=self.attn.num_heads, preferred_attention=self.attn.comfy_attention,
)
- attn2, _ = optimized_attention(
+ attn2 = optimized_attention(
qkv2[0], qkv2[1], qkv2[2],
- num_heads=self.attn2.num_heads,
+ heads=self.attn2.num_heads, preferred_attention=self.attn2.comfy_attention,
)
return self.post_attention_x(attn, attn2, *intermediates)
else:
qkv, intermediates = self.pre_attention(x, c)
+ qkv = tuple(AttentionTensorContainer(t) for t in qkv)
attn = optimized_attention(
qkv[0], qkv[1], qkv[2],
- heads=self.attn.num_heads,
+ heads=self.attn.num_heads, preferred_attention=self.attn.comfy_attention,
)
return self.post_attention(attn, *intermediates)
@@ -617,19 +622,19 @@ def _block_mixing(context, x, context_block, x_block, c, transformer_options={})
else:
x_qkv, x_intermediates = x_block.pre_attention(x, c)
- o = []
- for t in range(3):
- o.append(torch.cat((context_qkv[t], x_qkv[t]), dim=1))
- qkv = tuple(o)
+ context_len = context_qkv[0].shape[1]
+ qkv = tuple(AttentionTensorContainer(torch.cat((context_qkv[t], x_qkv[t]), dim=1)) for t in range(3))
+ del context_qkv, x_qkv
attn = optimized_attention(
qkv[0], qkv[1], qkv[2],
heads=x_block.attn.num_heads,
+ preferred_attention=x_block.attn.comfy_attention,
transformer_options=transformer_options,
)
context_attn, x_attn = (
- attn[:, : context_qkv[0].shape[1]],
- attn[:, context_qkv[0].shape[1] :],
+ attn[:, :context_len],
+ attn[:, context_len:],
)
if not context_block.pre_only:
@@ -638,9 +643,11 @@ def _block_mixing(context, x, context_block, x_block, c, transformer_options={})
else:
context = None
if x_block.x_block_self_attn:
+ x_qkv2 = tuple(AttentionTensorContainer(t) for t in x_qkv2)
attn2 = optimized_attention(
x_qkv2[0], x_qkv2[1], x_qkv2[2],
heads=x_block.attn2.num_heads,
+ preferred_attention=x_block.attn2.comfy_attention,
transformer_options=transformer_options,
)
x = x_block.post_attention_x(x_attn, attn2, *x_intermediates)
@@ -709,6 +716,7 @@ class FinalLayer(nn.Module):
class SelfAttentionContext(nn.Module):
def __init__(self, dim, heads=8, dim_head=64, dtype=None, device=None, operations=None):
super().__init__()
+ self.comfy_attention = ComfyAttention()
dim_head = dim // heads
inner_dim = dim
@@ -722,7 +730,10 @@ class SelfAttentionContext(nn.Module):
def forward(self, x):
qkv = self.qkv(x)
q, k, v = split_qkv(qkv, self.dim_head)
- x = optimized_attention(q.reshape(q.shape[0], q.shape[1], -1), k, v, heads=self.heads)
+ q = AttentionTensorContainer(q.reshape(q.shape[0], q.shape[1], -1))
+ k, v = AttentionTensorContainer(k), AttentionTensorContainer(v)
+ del qkv
+ x = optimized_attention(q, k, v, heads=self.heads, preferred_attention=self.comfy_attention)
return self.proj(x)
class ContextProcessorBlock(nn.Module):
diff --git a/comfy/ldm/seedvr/attention.py b/comfy/ldm/seedvr/attention.py
index a39f0ba9d..ca0a65c40 100644
--- a/comfy/ldm/seedvr/attention.py
+++ b/comfy/ldm/seedvr/attention.py
@@ -34,6 +34,8 @@ def _equal_length_runs(lengths):
def var_attention_optimized_split(q, k, v, heads, cu_seqlens_q, cu_seqlens_k, *args, skip_reshape=False, skip_output_reshape=False, **kwargs):
+ if isinstance(q, _attention.AttentionTensorContainer):
+ q, k, v = q.take(), k.take(), v.take()
q, k, v, head_dim = _var_attention_qkv(q, k, v, heads, skip_reshape)
if k.shape[0] != v.shape[0]:
@@ -54,7 +56,10 @@ def var_attention_optimized_split(q, k, v, heads, cu_seqlens_q, cu_seqlens_k, *a
q_i = q[q_slice].reshape(n, len_q, heads, head_dim).transpose(1, 2)
k_i = k[k_slice].reshape(n, len_k, heads, head_dim).transpose(1, 2)
v_i = v[k_slice].reshape(n, len_k, heads, head_dim).transpose(1, 2)
- out_i = _attention.optimized_attention(q_i, k_i, v_i, heads, skip_reshape=True, skip_output_reshape=True)
+ if last == len(q_lens):
+ del q, k, v
+ q_i, k_i, v_i = _attention.AttentionTensorContainer(q_i), _attention.AttentionTensorContainer(k_i), _attention.AttentionTensorContainer(v_i)
+ out_i = _attention.optimized_attention(q_i, k_i, v_i, heads, skip_reshape=True, skip_output_reshape=True, **kwargs)
out[q_slice] = out_i.transpose(1, 2).reshape(n * len_q, heads, head_dim)
return _var_attention_output(out, heads, head_dim, skip_output_reshape)
diff --git a/comfy/ldm/seedvr/model.py b/comfy/ldm/seedvr/model.py
index 16bbaa494..7b3a9071f 100644
--- a/comfy/ldm/seedvr/model.py
+++ b/comfy/ldm/seedvr/model.py
@@ -3,6 +3,7 @@ from typing import Optional, Tuple, Union, List, Dict, Any, Callable
import torch.nn.functional as F
from math import ceil, pi
import torch
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention
from itertools import accumulate, chain
from comfy.ldm.modules.diffusionmodules.model import get_timestep_embedding
from comfy.ldm.seedvr.attention import optimized_var_attention
@@ -488,6 +489,7 @@ class NaMMAttention(nn.Module):
device, dtype, operations,
):
super().__init__()
+ self.comfy_attention = ComfyAttention()
dim = MMArg(vid_dim, txt_dim)
self.heads = heads
inner_dim = heads * head_dim
@@ -597,6 +599,7 @@ class NaSwinAttention(NaMMAttention):
vid_q, vid_k, vid_v = vid_qkv_win.unbind(1)
txt_q, txt_k, txt_v = txt_qkv.unbind(1)
+ del txt_qkv, vid_qkv_win
vid_q, txt_q = self.norm_q(vid_q, txt_q)
vid_k, txt_k = self.norm_k(vid_k, txt_k)
@@ -624,14 +627,14 @@ class NaSwinAttention(NaMMAttention):
concat_win, unconcat_win, cu_seqlens_win = cache_win(
"mm_pnp", lambda: repeat_concat_idx(vid_len_win, txt_len, window_count)
)
- q = concat_win(vid_q, txt_q)
+ q = AttentionTensorContainer(concat_win(vid_q, txt_q))
del vid_q, txt_q
- k = concat_win(vid_k, txt_k)
+ k = AttentionTensorContainer(concat_win(vid_k, txt_k))
del vid_k, txt_k
- v = concat_win(vid_v, txt_v)
- del vid_v, txt_v, vid_qkv_win, txt_qkv
+ v = AttentionTensorContainer(concat_win(vid_v, txt_v))
+ del vid_v, txt_v
out = optimized_var_attention(
- q=q, k=k, v=v,
+ q=q, k=k, v=v, preferred_attention=self.comfy_attention,
heads=self.heads, skip_reshape=True, skip_output_reshape=True,
cu_seqlens_q=cu_seqlens_win,
cu_seqlens_k=cu_seqlens_win,
diff --git a/comfy/ldm/wan/ar_model.py b/comfy/ldm/wan/ar_model.py
index d72f53602..ecd05eb1f 100644
--- a/comfy/ldm/wan/ar_model.py
+++ b/comfy/ldm/wan/ar_model.py
@@ -12,7 +12,7 @@ Reference: https://github.com/thu-ml/Causal-Forcing
import torch
import torch.nn as nn
-from comfy.ldm.modules.attention import optimized_attention
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attention
from comfy.ldm.flux.math import apply_rope1
from comfy.ldm.wan.model import (
sinusoidal_embedding_1d,
@@ -31,6 +31,7 @@ class CausalWanSelfAttention(nn.Module):
eps=1e-6, operation_settings={}):
assert dim % num_heads == 0
super().__init__()
+ self.comfy_attention = ComfyAttention()
self.dim = dim
self.num_heads = num_heads
self.head_dim = dim // num_heads
@@ -55,15 +56,7 @@ class CausalWanSelfAttention(nn.Module):
k = apply_rope1(self.norm_k(self.k(x)).view(b, s, n, d), freqs)
v = self.v(x).view(b, s, n, d)
- if kv_cache is None:
- x = optimized_attention(
- q.view(b, s, n * d),
- k.view(b, s, n * d),
- v.view(b, s, n * d),
- heads=self.num_heads,
- transformer_options=transformer_options,
- )
- else:
+ if kv_cache is not None:
end = kv_cache["end"]
new_end = end + s
@@ -72,13 +65,17 @@ class CausalWanSelfAttention(nn.Module):
kv_cache["v"][:, end:new_end] = v
kv_cache["end"] = new_end
- x = optimized_attention(
- q.view(b, s, n * d),
- kv_cache["k"][:, :new_end].view(b, new_end, n * d),
- kv_cache["v"][:, :new_end].view(b, new_end, n * d),
- heads=self.num_heads,
- transformer_options=transformer_options,
- )
+ k = kv_cache["k"][:, :new_end]
+ v = kv_cache["v"][:, :new_end]
+
+ q = AttentionTensorContainer(q.view(b, s, n * d))
+ k = AttentionTensorContainer(k.reshape(b, -1, n * d))
+ v = AttentionTensorContainer(v.reshape(b, -1, n * d))
+ x = optimized_attention(
+ q, k, v, heads=self.num_heads,
+ preferred_attention=self.comfy_attention,
+ transformer_options=transformer_options,
+ )
x = self.o(x)
return x
@@ -114,10 +111,10 @@ class CausalWanAttentionBlock(WanAttentionBlock):
# Cross-attention with optional caching
if crossattn_cache is not None and crossattn_cache.get("is_init"):
- q = self.cross_attn.norm_q(self.cross_attn.q(self.norm3(x)))
+ q = AttentionTensorContainer(self.cross_attn.norm_q(self.cross_attn.q(self.norm3(x))))
x_ca = optimized_attention(
- q, crossattn_cache["k"], crossattn_cache["v"],
- heads=self.num_heads, transformer_options=transformer_options)
+ q, AttentionTensorContainer(crossattn_cache["k"]), AttentionTensorContainer(crossattn_cache["v"]),
+ heads=self.num_heads, preferred_attention=self.cross_attn.comfy_attention, transformer_options=transformer_options)
x = x + self.cross_attn.o(x_ca)
else:
x = x + self.cross_attn(self.norm3(x), context, context_img_len=context_img_len, transformer_options=transformer_options)
diff --git a/tests-unit/comfy_test/test_seedvr2_internals.py b/tests-unit/comfy_test/test_seedvr2_internals.py
index 9e620a215..431eec281 100644
--- a/tests-unit/comfy_test/test_seedvr2_internals.py
+++ b/tests-unit/comfy_test/test_seedvr2_internals.py
@@ -125,8 +125,8 @@ def test_seedvr2_7b_swin_attention_forward_uses_optimized_var_attention(monkeypa
calls = []
def fake_optimized_var_attention(**kwargs):
- calls.append(kwargs)
- return kwargs["q"]
+ calls.append({**kwargs, **{name: kwargs[name].take() for name in ("q", "k", "v")}})
+ return calls[-1]["q"]
monkeypatch.setattr(seedvr_model, "optimized_var_attention", fake_optimized_var_attention)
@@ -136,6 +136,7 @@ def test_seedvr2_7b_swin_attention_forward_uses_optimized_var_attention(monkeypa
assert tuple(txt_out.shape) == (3, dim)
assert len(calls) == 1
call = calls[0]
+ assert call["preferred_attention"] is attn.comfy_attention
assert tuple(call["q"].shape) == (14, heads, head_dim)
assert tuple(call["k"].shape) == (14, heads, head_dim)
assert tuple(call["v"].shape) == (14, heads, head_dim)
@@ -195,7 +196,8 @@ def test_seedvr2_swin_attention_batched_samples_match_one_at_a_time(rope_type):
torch.testing.assert_close(batched_txt, torch.cat(single_txt), rtol=1e-4, atol=1e-5)
-def test_var_attention_optimized_split_batches_equal_length_windows(monkeypatch):
[email protected]("container_inputs", [False, True])
+def test_var_attention_optimized_split_batches_equal_length_windows(monkeypatch, container_inputs):
heads = 2
head_dim = 3
q = torch.arange(36, dtype=torch.float32).reshape(6, heads, head_dim)
@@ -203,14 +205,20 @@ def test_var_attention_optimized_split_batches_equal_length_windows(monkeypatch)
v = q + 200
cu = [0, 2, 4, 6]
calls = []
+ preference = attention.ComfyAttention()
+ options = {"block_index": 2}
def fake_optimized_attention(q_arg, k_arg, v_arg, heads_arg, **kwargs):
+ assert kwargs["preferred_attention"] is preference
+ assert kwargs["transformer_options"] is options
+ q_arg, k_arg, v_arg = q_arg.take(), k_arg.take(), v_arg.take()
calls.append(tuple(q_arg.shape))
return q_arg + v_arg
monkeypatch.setattr(attention, "optimized_attention", fake_optimized_attention)
- out = var_attention_optimized_split(q, k, v, heads, cu, cu, skip_reshape=True, skip_output_reshape=True)
+ inputs = tuple(attention.AttentionTensorContainer(t) for t in (q, k, v)) if container_inputs else (q, k, v)
+ out = var_attention_optimized_split(*inputs, heads, cu, cu, skip_reshape=True, skip_output_reshape=True, preferred_attention=preference, transformer_options=options)
assert calls == [(3, heads, 2, head_dim)], (
f"equal-length windows must share one batched attention call; got {calls}"
@@ -228,6 +236,7 @@ def test_var_attention_optimized_split_calls_dense_backend_per_window(monkeypatc
calls = []
def fake_optimized_attention(q_arg, k_arg, v_arg, heads_arg, **kwargs):
+ q_arg, k_arg, v_arg = q_arg.take(), k_arg.take(), v_arg.take()
calls.append(
{
"q_shape": tuple(q_arg.shape),