The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Add comfy_attention and AttentionTensorContainer support to rest of models (#16806)
comfyanonymousdddb41aad74e
diff --git a/comfy/ldm/hidream/model.py b/comfy/ldm/hidream/model.pyindex 28d81c79e..85f278197 100644--- a/comfy/ldm/hidream/model.py+++ b/comfy/ldm/hidream/model.py@@ -11,7 +11,7 @@ import torch.nn.functional as Ffrom comfy.ldm.flux.math import apply_rope, ropefrom comfy.ldm.flux.layers import LastLayer-from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attentionimport comfy.model_managementimport comfy.patcher_extensionimport comfy.ldm.common_dit@@ -72,8 +72,12 @@ class TimestepEmbed(nn.Module):return t_emb-def attention(query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, transformer_options={}):- return optimized_attention(query.view(query.shape[0], -1, query.shape[-1] * query.shape[-2]), key.view(key.shape[0], -1, key.shape[-1] * key.shape[-2]), value.view(value.shape[0], -1, value.shape[-1] * value.shape[-2]), query.shape[2], transformer_options=transformer_options)+def attention(query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, transformer_options={}, preferred_attention=None):+ if isinstance(query, AttentionTensorContainer):+ query, key, value = query.take(), key.take(), value.take()+ heads = query.shape[2]+ query, key, value = AttentionTensorContainer(query.flatten(2)), AttentionTensorContainer(key.flatten(2)), AttentionTensorContainer(value.flatten(2))+ return optimized_attention(query, key, value, heads, preferred_attention=preferred_attention, transformer_options=transformer_options)class HiDreamAttnProcessor_flashattn:@@ -120,11 +124,13 @@ class HiDreamAttnProcessor_flashattn:query = torch.cat([query_i, query_t], dim=1)key = torch.cat([key_i, key_t], dim=1)value = torch.cat([value_i, value_t], dim=1)+ del query_t, key_t, value_telse:query = query_ikey = key_ivalue = value_i+ del query_i, key_i, value_iif query.shape[-1] == rope.shape[-3] * 2:query, key = apply_rope(query, key, rope)else:@@ -133,8 +139,10 @@ class HiDreamAttnProcessor_flashattn:query_1, key_1 = apply_rope(query_1, key_1, rope)query = torch.cat([query_1, query_2], dim=-1)key = torch.cat([key_1, key_2], dim=-1)+ del query_1, query_2, key_1, key_2- hidden_states = attention(query, key, value, transformer_options=transformer_options)+ query, key, value = AttentionTensorContainer(query), AttentionTensorContainer(key), AttentionTensorContainer(value)+ hidden_states = attention(query, key, value, transformer_options=transformer_options, preferred_attention=attn.comfy_attention)if not attn.single:hidden_states_i, hidden_states_t = torch.split(hidden_states, [num_image_tokens, num_text_tokens], dim=1)@@ -162,6 +170,7 @@ class HiDreamAttention(nn.Module):):# super(Attention, self).__init__()super().__init__()+ self.comfy_attention = ComfyAttention()self.inner_dim = out_dim if out_dim is not None else dim_head * headsself.query_dim = query_dimself.upcast_attention = upcast_attentiondiff --git a/comfy/ldm/hidream_o1/attention.py b/comfy/ldm/hidream_o1/attention.pyindex afb2be9b8..b857424c7 100644--- a/comfy/ldm/hidream_o1/attention.py+++ b/comfy/ldm/hidream_o1/attention.py@@ -7,10 +7,10 @@ gen half hit the user's preferred backend via optimized_attention.import torchimport comfy.ops-from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.modules.attention import AttentionTensorContainer, optimized_attention-def make_two_pass_attention(ar_len: int, transformer_options=None):+def make_two_pass_attention(ar_len: int, transformer_options=None, preferred_attention=None):"""Build a two-pass attention callable. AR pass uses SDPA-causal directly, gen pass routes through optimized_attention.The AR pass goes through SDPA directand bypasses wrappers, it is only ~1% of T at typical edit sizes."""@@ -19,20 +19,23 @@ def make_two_pass_attention(ar_len: int, transformer_options=None):B, H, T, D = q.shapeif T < k.shape[2]: # KV-cache hot path: Q is shorter than K/V (cached AR prefix is in K/V only), all fresh Q positions are in the gen region, single full-attention call- out = optimized_attention(q, k, v, heads, mask=None, skip_reshape=True, skip_output_reshape=True, transformer_options=transformer_options, enable_gqa=enable_gqa)+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)+ out = optimized_attention(q, k, v, heads, mask=None, skip_reshape=True, skip_output_reshape=True, transformer_options=transformer_options, preferred_attention=preferred_attention, enable_gqa=enable_gqa)elif ar_len >= T:out = comfy.ops.scaled_dot_product_attention(q, k, v, attn_mask=None, dropout_p=0.0, is_causal=True, enable_gqa=enable_gqa)elif ar_len <= 0:- out = optimized_attention(q, k, v, heads, mask=None, skip_reshape=True, skip_output_reshape=True, transformer_options=transformer_options, enable_gqa=enable_gqa)+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)+ out = optimized_attention(q, k, v, heads, mask=None, skip_reshape=True, skip_output_reshape=True, transformer_options=transformer_options, preferred_attention=preferred_attention, enable_gqa=enable_gqa)else:out_ar = comfy.ops.scaled_dot_product_attention(q[:, :, :ar_len], k[:, :, :ar_len], v[:, :, :ar_len],attn_mask=None, dropout_p=0.0, is_causal=True, enable_gqa=enable_gqa,)+ q, k, v = AttentionTensorContainer(q[:, :, ar_len:]), AttentionTensorContainer(k), AttentionTensorContainer(v)out_gen = optimized_attention(- q[:, :, ar_len:], k, v, heads,+ q, k, v, heads,mask=None, skip_reshape=True, skip_output_reshape=True,- transformer_options=transformer_options, enable_gqa=enable_gqa,+ transformer_options=transformer_options, preferred_attention=preferred_attention, enable_gqa=enable_gqa,)out = torch.cat([out_ar, out_gen], dim=2)diff --git a/comfy/ldm/hidream_o1/model.py b/comfy/ldm/hidream_o1/model.pyindex a223e706f..03d2a73d0 100644--- a/comfy/ldm/hidream_o1/model.py+++ b/comfy/ldm/hidream_o1/model.py@@ -15,6 +15,7 @@ import torchimport torch.nn as nnimport comfy.patcher_extension+from comfy.ldm.modules.attention import ComfyAttentionfrom comfy.ldm.modules.diffusionmodules.mmdit import TimestepEmbedderfrom comfy.text_encoders.llama import Llama2_from comfy.text_encoders.qwen35 import Qwen35VisionModel@@ -113,6 +114,8 @@ class HiDreamO1Transformer(nn.Module):self.visual = Qwen35VisionModel(vision_cfg, device=device, dtype=dtype, ops=operations)self.language_model = Llama2_(text_cfg, device=device, dtype=dtype, ops=operations)+ for layer in self.language_model.layers:+ layer.self_attn.comfy_attention = ComfyAttention()self.t_embedder1 = TimestepEmbedder(text_cfg.hidden_size, device=device, dtype=dtype, operations=operations,)@@ -197,7 +200,6 @@ class HiDreamO1Transformer(nn.Module):freqs_cis = self.language_model.compute_freqs_cis(position_ids[0].to(x.device), x.device)freqs_cis = tuple(t.to(x.dtype) for t in freqs_cis)- two_pass_attn = make_two_pass_attention(ar_len, transformer_options=transformer_options)patches_replace = transformer_options.get("patches_replace", {})blocks_replace = patches_replace.get("dit", {})transformer_options["total_blocks"] = len(self.language_model.layers)@@ -231,6 +233,7 @@ class HiDreamO1Transformer(nn.Module):hidden_states = inputs_embeds[:, cache_len:]sliced_freqs = tuple(t[..., cache_len:, :] for t in freqs_cis)for i, layer in enumerate(self.language_model.layers):+ two_pass_attn = make_two_pass_attention(ar_len, transformer_options=transformer_options, preferred_attention=layer.self_attn.comfy_attention)transformer_options["block_index"] = iK_i, V_i = kv_cache["kv"][i]hidden_states, _ = layer(@@ -243,6 +246,7 @@ class HiDreamO1Transformer(nn.Module):past_kv_cold = () if can_cache else Nonehidden_states = inputs_embedsfor i, layer in enumerate(self.language_model.layers):+ two_pass_attn = make_two_pass_attention(ar_len, transformer_options=transformer_options, preferred_attention=layer.self_attn.comfy_attention)transformer_options["block_index"] = iif ("double_block", i) in blocks_replace:def block_wrap(args, _layer=layer):diff --git a/comfy/ldm/hunyuan3dv2_1/hunyuandit.py b/comfy/ldm/hunyuan3dv2_1/hunyuandit.pyindex 4e4819fe3..32ca98503 100644--- a/comfy/ldm/hunyuan3dv2_1/hunyuandit.py+++ b/comfy/ldm/hunyuan3dv2_1/hunyuandit.py@@ -2,7 +2,7 @@ import mathimport torchimport torch.nn as nnimport torch.nn.functional as F-from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attentionimport comfy.model_managementclass GELU(nn.Module):@@ -288,6 +288,7 @@ class CrossAttention(nn.Module):**kwargs,):super().__init__()+ self.comfy_attention = ComfyAttention()self.qdim = qdimself.kdim = kdim@@ -338,12 +339,14 @@ class CrossAttention(nn.Module):q = self.q_norm(q)k = self.k_norm(k)+ q, k, v = AttentionTensorContainer(q.reshape(b, s1, -1)), AttentionTensorContainer(k.reshape(b, s2, -1)), AttentionTensorContainer(v)+ del kvx = optimized_attention(- q.reshape(b, s1, self.num_heads * self.head_dim),- k.reshape(b, s2, self.num_heads * self.head_dim),+ q,+ k,v,heads=self.num_heads,- low_precision_attention=False,+ low_precision_attention=False, preferred_attention=self.comfy_attention,)out = self.out_proj(x)@@ -365,6 +368,7 @@ class Attention(nn.Module):dtype = None):super().__init__()+ self.comfy_attention = ComfyAttention()self.dim = dimself.num_heads = num_headsself.head_dim = self.dim // num_heads@@ -408,12 +412,14 @@ class Attention(nn.Module):query = self.q_norm(query)key = self.k_norm(key)+ query, key, value = AttentionTensorContainer(query.reshape(B, N, -1)), AttentionTensorContainer(key.reshape(B, N, -1)), AttentionTensorContainer(value)+ del qkv, qkv_combinedx = optimized_attention(- query.reshape(B, N, self.num_heads * self.head_dim),- key.reshape(B, N, self.num_heads * self.head_dim),+ query,+ key,value,heads=self.num_heads,- low_precision_attention=False,+ low_precision_attention=False, preferred_attention=self.comfy_attention,)x = self.out_proj(x)diff --git a/comfy/ldm/lightricks/model.py b/comfy/ldm/lightricks/model.pyindex dcbfa43ad..b011f5d81 100644--- a/comfy/ldm/lightricks/model.py+++ b/comfy/ldm/lightricks/model.py@@ -11,6 +11,7 @@ import torchfrom torch import nnimport comfy.patcher_extensionimport comfy.ldm.modules.attention+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttentionimport comfy.ldm.common_ditimport comfy.model_managementimport comfy.ops@@ -393,7 +394,7 @@ class GuideAttentionMask:self.tracked_mask[:, :, :, :guide_start] = log_w.view(1, 1, -1, 1)-def _attention_with_guide_mask(q, k, v, heads, guide_mask, attn_precision, transformer_options):+def _attention_with_guide_mask(q, k, v, heads, guide_mask, attn_precision, transformer_options, preferred_attention=None):"""Apply the guide mask by partitioning Q into noisy and tracked-guidegroups, so each group needs only its own sub-mask. Avoids materializingthe (1,1,T,T) dense mask.@@ -405,19 +406,19 @@ def _attention_with_guide_mask(q, k, v, heads, guide_mask, attn_precision, transif guide_start > 0: # In practice currently guides are always after noise, guard for safety if this changes.out[:, :guide_start, :] = comfy.ldm.modules.attention.optimized_attention(- q[:, :guide_start, :], k, v, heads, mask=guide_mask.noisy_mask,- attn_precision=attn_precision, transformer_options=transformer_options,+ AttentionTensorContainer(q[:, :guide_start, :]), AttentionTensorContainer(k), AttentionTensorContainer(v), heads, mask=guide_mask.noisy_mask,+ attn_precision=attn_precision, transformer_options=transformer_options, preferred_attention=preferred_attention,low_precision_attention=False, # sageattn mask support is unreliable)out[:, guide_start:tracked_end, :] = comfy.ldm.modules.attention.optimized_attention(- q[:, guide_start:tracked_end, :], k, v, heads, mask=guide_mask.tracked_mask,- attn_precision=attn_precision, transformer_options=transformer_options,+ AttentionTensorContainer(q[:, guide_start:tracked_end, :]), AttentionTensorContainer(k), AttentionTensorContainer(v), heads, mask=guide_mask.tracked_mask,+ attn_precision=attn_precision, transformer_options=transformer_options, preferred_attention=preferred_attention,low_precision_attention=False,)if tracked_end < q.shape[1]: # Every guide token is tracked, and nothing comes after them, guard for safety if this changes.out[:, tracked_end:, :] = comfy.ldm.modules.attention.optimized_attention(- q[:, tracked_end:, :], k, v, heads,- attn_precision=attn_precision, transformer_options=transformer_options,+ AttentionTensorContainer(q[:, tracked_end:, :]), AttentionTensorContainer(k), AttentionTensorContainer(v), heads,+ attn_precision=attn_precision, transformer_options=transformer_options, preferred_attention=preferred_attention,)return out@@ -437,6 +438,7 @@ class CrossAttention(nn.Module):operations=None,):super().__init__()+ self.comfy_attention = ComfyAttention()inner_dim = dim_head * headscontext_dim = query_dim if context_dim is None else context_dimself.attn_precision = attn_precision@@ -484,12 +486,11 @@ class CrossAttention(nn.Module):q = apply_rotary_emb(q, pe)k = apply_rotary_emb(k, pe if k_pe is None else k_pe)- if mask is None:- out = comfy.ldm.modules.attention.optimized_attention(q, k, v, self.heads, attn_precision=self.attn_precision, transformer_options=transformer_options)- elif isinstance(mask, GuideAttentionMask):- out = _attention_with_guide_mask(q, k, v, self.heads, mask, attn_precision=self.attn_precision, transformer_options=transformer_options)+ if isinstance(mask, GuideAttentionMask):+ out = _attention_with_guide_mask(q, k, v, self.heads, mask, attn_precision=self.attn_precision, transformer_options=transformer_options, preferred_attention=self.comfy_attention)else:- out = comfy.ldm.modules.attention.optimized_attention(q, k, v, self.heads, mask=mask, attn_precision=self.attn_precision, transformer_options=transformer_options)+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)+ out = comfy.ldm.modules.attention.optimized_attention(q, k, v, self.heads, mask=mask, attn_precision=self.attn_precision, transformer_options=transformer_options, preferred_attention=self.comfy_attention)# Apply per-head gating if enabledif self.to_gate_logits is not None:diff --git a/comfy/ldm/modules/diffusionmodules/mmdit.py b/comfy/ldm/modules/diffusionmodules/mmdit.pyindex 9ab3c463c..99407233b 100644--- a/comfy/ldm/modules/diffusionmodules/mmdit.py+++ b/comfy/ldm/modules/diffusionmodules/mmdit.py@@ -4,7 +4,7 @@ from typing import Dict, Optional, Listimport numpy as npimport torchimport torch.nn as nn-from ..attention import optimized_attention+from ..attention import AttentionTensorContainer, ComfyAttention, optimized_attentionfrom einops import rearrange, repeatfrom .util import timestep_embeddingimport comfy.ops@@ -276,6 +276,7 @@ class SelfAttention(nn.Module):operations=None,):super().__init__()+ self.comfy_attention = ComfyAttention()self.num_heads = num_headsself.head_dim = dim // num_heads@@ -315,8 +316,9 @@ class SelfAttention(nn.Module):def forward(self, x: torch.Tensor) -> torch.Tensor:q, k, v = self.pre_attention(x)+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)x = optimized_attention(- q, k, v, heads=self.num_heads+ q, k, v, heads=self.num_heads, preferred_attention=self.comfy_attention)x = self.post_attention(x)return x@@ -577,20 +579,23 @@ class DismantledBlock(nn.Module):assert not self.pre_onlyif self.x_block_self_attn:qkv, qkv2, intermediates = self.pre_attention_x(x, c)- attn, _ = optimized_attention(+ qkv = tuple(AttentionTensorContainer(t) for t in qkv)+ qkv2 = tuple(AttentionTensorContainer(t) for t in qkv2)+ attn = optimized_attention(qkv[0], qkv[1], qkv[2],- num_heads=self.attn.num_heads,+ heads=self.attn.num_heads, preferred_attention=self.attn.comfy_attention,)- attn2, _ = optimized_attention(+ attn2 = optimized_attention(qkv2[0], qkv2[1], qkv2[2],- num_heads=self.attn2.num_heads,+ heads=self.attn2.num_heads, preferred_attention=self.attn2.comfy_attention,)return self.post_attention_x(attn, attn2, *intermediates)else:qkv, intermediates = self.pre_attention(x, c)+ qkv = tuple(AttentionTensorContainer(t) for t in qkv)attn = optimized_attention(qkv[0], qkv[1], qkv[2],- heads=self.attn.num_heads,+ heads=self.attn.num_heads, preferred_attention=self.attn.comfy_attention,)return self.post_attention(attn, *intermediates)@@ -617,19 +622,19 @@ def _block_mixing(context, x, context_block, x_block, c, transformer_options={})else:x_qkv, x_intermediates = x_block.pre_attention(x, c)- o = []- for t in range(3):- o.append(torch.cat((context_qkv[t], x_qkv[t]), dim=1))- qkv = tuple(o)+ context_len = context_qkv[0].shape[1]+ qkv = tuple(AttentionTensorContainer(torch.cat((context_qkv[t], x_qkv[t]), dim=1)) for t in range(3))+ del context_qkv, x_qkvattn = optimized_attention(qkv[0], qkv[1], qkv[2],heads=x_block.attn.num_heads,+ preferred_attention=x_block.attn.comfy_attention,transformer_options=transformer_options,)context_attn, x_attn = (- attn[:, : context_qkv[0].shape[1]],- attn[:, context_qkv[0].shape[1] :],+ attn[:, :context_len],+ attn[:, context_len:],)if not context_block.pre_only:@@ -638,9 +643,11 @@ def _block_mixing(context, x, context_block, x_block, c, transformer_options={})else:context = Noneif x_block.x_block_self_attn:+ x_qkv2 = tuple(AttentionTensorContainer(t) for t in x_qkv2)attn2 = optimized_attention(x_qkv2[0], x_qkv2[1], x_qkv2[2],heads=x_block.attn2.num_heads,+ preferred_attention=x_block.attn2.comfy_attention,transformer_options=transformer_options,)x = x_block.post_attention_x(x_attn, attn2, *x_intermediates)@@ -709,6 +716,7 @@ class FinalLayer(nn.Module):class SelfAttentionContext(nn.Module):def __init__(self, dim, heads=8, dim_head=64, dtype=None, device=None, operations=None):super().__init__()+ self.comfy_attention = ComfyAttention()dim_head = dim // headsinner_dim = dim@@ -722,7 +730,10 @@ class SelfAttentionContext(nn.Module):def forward(self, x):qkv = self.qkv(x)q, k, v = split_qkv(qkv, self.dim_head)- x = optimized_attention(q.reshape(q.shape[0], q.shape[1], -1), k, v, heads=self.heads)+ q = AttentionTensorContainer(q.reshape(q.shape[0], q.shape[1], -1))+ k, v = AttentionTensorContainer(k), AttentionTensorContainer(v)+ del qkv+ x = optimized_attention(q, k, v, heads=self.heads, preferred_attention=self.comfy_attention)return self.proj(x)class ContextProcessorBlock(nn.Module):diff --git a/comfy/ldm/seedvr/attention.py b/comfy/ldm/seedvr/attention.pyindex a39f0ba9d..ca0a65c40 100644--- a/comfy/ldm/seedvr/attention.py+++ b/comfy/ldm/seedvr/attention.py@@ -34,6 +34,8 @@ def _equal_length_runs(lengths):def var_attention_optimized_split(q, k, v, heads, cu_seqlens_q, cu_seqlens_k, *args, skip_reshape=False, skip_output_reshape=False, **kwargs):+ if isinstance(q, _attention.AttentionTensorContainer):+ q, k, v = q.take(), k.take(), v.take()q, k, v, head_dim = _var_attention_qkv(q, k, v, heads, skip_reshape)if k.shape[0] != v.shape[0]:@@ -54,7 +56,10 @@ def var_attention_optimized_split(q, k, v, heads, cu_seqlens_q, cu_seqlens_k, *aq_i = q[q_slice].reshape(n, len_q, heads, head_dim).transpose(1, 2)k_i = k[k_slice].reshape(n, len_k, heads, head_dim).transpose(1, 2)v_i = v[k_slice].reshape(n, len_k, heads, head_dim).transpose(1, 2)- out_i = _attention.optimized_attention(q_i, k_i, v_i, heads, skip_reshape=True, skip_output_reshape=True)+ if last == len(q_lens):+ del q, k, v+ q_i, k_i, v_i = _attention.AttentionTensorContainer(q_i), _attention.AttentionTensorContainer(k_i), _attention.AttentionTensorContainer(v_i)+ out_i = _attention.optimized_attention(q_i, k_i, v_i, heads, skip_reshape=True, skip_output_reshape=True, **kwargs)out[q_slice] = out_i.transpose(1, 2).reshape(n * len_q, heads, head_dim)return _var_attention_output(out, heads, head_dim, skip_output_reshape)diff --git a/comfy/ldm/seedvr/model.py b/comfy/ldm/seedvr/model.pyindex 16bbaa494..7b3a9071f 100644--- a/comfy/ldm/seedvr/model.py+++ b/comfy/ldm/seedvr/model.py@@ -3,6 +3,7 @@ from typing import Optional, Tuple, Union, List, Dict, Any, Callableimport torch.nn.functional as Ffrom math import ceil, piimport torch+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttentionfrom itertools import accumulate, chainfrom comfy.ldm.modules.diffusionmodules.model import get_timestep_embeddingfrom comfy.ldm.seedvr.attention import optimized_var_attention@@ -488,6 +489,7 @@ class NaMMAttention(nn.Module):device, dtype, operations,):super().__init__()+ self.comfy_attention = ComfyAttention()dim = MMArg(vid_dim, txt_dim)self.heads = headsinner_dim = heads * head_dim@@ -597,6 +599,7 @@ class NaSwinAttention(NaMMAttention):vid_q, vid_k, vid_v = vid_qkv_win.unbind(1)txt_q, txt_k, txt_v = txt_qkv.unbind(1)+ del txt_qkv, vid_qkv_winvid_q, txt_q = self.norm_q(vid_q, txt_q)vid_k, txt_k = self.norm_k(vid_k, txt_k)@@ -624,14 +627,14 @@ class NaSwinAttention(NaMMAttention):concat_win, unconcat_win, cu_seqlens_win = cache_win("mm_pnp", lambda: repeat_concat_idx(vid_len_win, txt_len, window_count))- q = concat_win(vid_q, txt_q)+ q = AttentionTensorContainer(concat_win(vid_q, txt_q))del vid_q, txt_q- k = concat_win(vid_k, txt_k)+ k = AttentionTensorContainer(concat_win(vid_k, txt_k))del vid_k, txt_k- v = concat_win(vid_v, txt_v)- del vid_v, txt_v, vid_qkv_win, txt_qkv+ v = AttentionTensorContainer(concat_win(vid_v, txt_v))+ del vid_v, txt_vout = optimized_var_attention(- q=q, k=k, v=v,+ q=q, k=k, v=v, preferred_attention=self.comfy_attention,heads=self.heads, skip_reshape=True, skip_output_reshape=True,cu_seqlens_q=cu_seqlens_win,cu_seqlens_k=cu_seqlens_win,diff --git a/comfy/ldm/wan/ar_model.py b/comfy/ldm/wan/ar_model.pyindex d72f53602..ecd05eb1f 100644--- a/comfy/ldm/wan/ar_model.py+++ b/comfy/ldm/wan/ar_model.py@@ -12,7 +12,7 @@ Reference: https://github.com/thu-ml/Causal-Forcingimport torchimport torch.nn as nn-from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention, optimized_attentionfrom comfy.ldm.flux.math import apply_rope1from comfy.ldm.wan.model import (sinusoidal_embedding_1d,@@ -31,6 +31,7 @@ class CausalWanSelfAttention(nn.Module):eps=1e-6, operation_settings={}):assert dim % num_heads == 0super().__init__()+ self.comfy_attention = ComfyAttention()self.dim = dimself.num_heads = num_headsself.head_dim = dim // num_heads@@ -55,15 +56,7 @@ class CausalWanSelfAttention(nn.Module):k = apply_rope1(self.norm_k(self.k(x)).view(b, s, n, d), freqs)v = self.v(x).view(b, s, n, d)- if kv_cache is None:- x = optimized_attention(- q.view(b, s, n * d),- k.view(b, s, n * d),- v.view(b, s, n * d),- heads=self.num_heads,- transformer_options=transformer_options,- )- else:+ if kv_cache is not None:end = kv_cache["end"]new_end = end + s@@ -72,13 +65,17 @@ class CausalWanSelfAttention(nn.Module):kv_cache["v"][:, end:new_end] = vkv_cache["end"] = new_end- x = optimized_attention(- q.view(b, s, n * d),- kv_cache["k"][:, :new_end].view(b, new_end, n * d),- kv_cache["v"][:, :new_end].view(b, new_end, n * d),- heads=self.num_heads,- transformer_options=transformer_options,- )+ k = kv_cache["k"][:, :new_end]+ v = kv_cache["v"][:, :new_end]++ q = AttentionTensorContainer(q.view(b, s, n * d))+ k = AttentionTensorContainer(k.reshape(b, -1, n * d))+ v = AttentionTensorContainer(v.reshape(b, -1, n * d))+ x = optimized_attention(+ q, k, v, heads=self.num_heads,+ preferred_attention=self.comfy_attention,+ transformer_options=transformer_options,+ )x = self.o(x)return x@@ -114,10 +111,10 @@ class CausalWanAttentionBlock(WanAttentionBlock):# Cross-attention with optional cachingif crossattn_cache is not None and crossattn_cache.get("is_init"):- q = self.cross_attn.norm_q(self.cross_attn.q(self.norm3(x)))+ q = AttentionTensorContainer(self.cross_attn.norm_q(self.cross_attn.q(self.norm3(x))))x_ca = optimized_attention(- q, crossattn_cache["k"], crossattn_cache["v"],- heads=self.num_heads, transformer_options=transformer_options)+ q, AttentionTensorContainer(crossattn_cache["k"]), AttentionTensorContainer(crossattn_cache["v"]),+ heads=self.num_heads, preferred_attention=self.cross_attn.comfy_attention, transformer_options=transformer_options)x = x + self.cross_attn.o(x_ca)else:x = x + self.cross_attn(self.norm3(x), context, context_img_len=context_img_len, transformer_options=transformer_options)diff --git a/tests-unit/comfy_test/test_seedvr2_internals.py b/tests-unit/comfy_test/test_seedvr2_internals.pyindex 9e620a215..431eec281 100644--- a/tests-unit/comfy_test/test_seedvr2_internals.py+++ b/tests-unit/comfy_test/test_seedvr2_internals.py@@ -125,8 +125,8 @@ def test_seedvr2_7b_swin_attention_forward_uses_optimized_var_attention(monkeypacalls = []def fake_optimized_var_attention(**kwargs):- calls.append(kwargs)- return kwargs["q"]+ calls.append({**kwargs, **{name: kwargs[name].take() for name in ("q", "k", "v")}})+ return calls[-1]["q"]monkeypatch.setattr(seedvr_model, "optimized_var_attention", fake_optimized_var_attention)@@ -136,6 +136,7 @@ def test_seedvr2_7b_swin_attention_forward_uses_optimized_var_attention(monkeypaassert tuple(txt_out.shape) == (3, dim)assert len(calls) == 1call = calls[0]+ assert call["preferred_attention"] is attn.comfy_attentionassert tuple(call["q"].shape) == (14, heads, head_dim)assert tuple(call["k"].shape) == (14, heads, head_dim)assert tuple(call["v"].shape) == (14, heads, head_dim)@@ -195,7 +196,8 @@ def test_seedvr2_swin_attention_batched_samples_match_one_at_a_time(rope_type):torch.testing.assert_close(batched_txt, torch.cat(single_txt), rtol=1e-4, atol=1e-5)-def test_var_attention_optimized_split_batches_equal_length_windows(monkeypatch):[email protected]("container_inputs", [False, True])+def test_var_attention_optimized_split_batches_equal_length_windows(monkeypatch, container_inputs):heads = 2head_dim = 3q = torch.arange(36, dtype=torch.float32).reshape(6, heads, head_dim)@@ -203,14 +205,20 @@ def test_var_attention_optimized_split_batches_equal_length_windows(monkeypatch)v = q + 200cu = [0, 2, 4, 6]calls = []+ preference = attention.ComfyAttention()+ options = {"block_index": 2}def fake_optimized_attention(q_arg, k_arg, v_arg, heads_arg, **kwargs):+ assert kwargs["preferred_attention"] is preference+ assert kwargs["transformer_options"] is options+ q_arg, k_arg, v_arg = q_arg.take(), k_arg.take(), v_arg.take()calls.append(tuple(q_arg.shape))return q_arg + v_argmonkeypatch.setattr(attention, "optimized_attention", fake_optimized_attention)- out = var_attention_optimized_split(q, k, v, heads, cu, cu, skip_reshape=True, skip_output_reshape=True)+ inputs = tuple(attention.AttentionTensorContainer(t) for t in (q, k, v)) if container_inputs else (q, k, v)+ out = var_attention_optimized_split(*inputs, heads, cu, cu, skip_reshape=True, skip_output_reshape=True, preferred_attention=preference, transformer_options=options)assert calls == [(3, heads, 2, head_dim)], (f"equal-length windows must share one batched attention call; got {calls}"@@ -228,6 +236,7 @@ def test_var_attention_optimized_split_calls_dense_backend_per_window(monkeypatccalls = []def fake_optimized_attention(q_arg, k_arg, v_arg, heads_arg, **kwargs):+ q_arg, k_arg, v_arg = q_arg.take(), k_arg.take(), v_arg.take()calls.append({"q_shape": tuple(q_arg.shape),