Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Add support for sage attention 3 in comfyui, enable via new cli arg (#11026)

mengqin0357ed7ec4a1
diff --git a/comfy/ldm/modules/attention.py b/comfy/ldm/modules/attention.py
index a8800ded0..ccf690945 100644
--- a/comfy/ldm/modules/attention.py
+++ b/comfy/ldm/modules/attention.py
@@ -30,6 +30,13 @@ except ImportError as e:
raise e
exit(-1)
+SAGE_ATTENTION3_IS_AVAILABLE = False
+try:
+ from sageattn3 import sageattn3_blackwell
+ SAGE_ATTENTION3_IS_AVAILABLE = True
+except ImportError:
+ pass
+
FLASH_ATTENTION_IS_AVAILABLE = False
try:
from flash_attn import flash_attn_func
@@ -563,6 +570,93 @@ def attention_sage(q, k, v, heads, mask=None, attn_precision=None, skip_reshape=
out = out.reshape(b, -1, heads * dim_head)
return out
+@wrap_attn
+def attention3_sage(q, k, v, heads, mask=None, attn_precision=None, skip_reshape=False, skip_output_reshape=False, **kwargs):
+ exception_fallback = False
+ if (q.device.type != "cuda" or
+ q.dtype not in (torch.float16, torch.bfloat16) or
+ mask is not None):
+ return attention_pytorch(
+ q, k, v, heads,
+ mask=mask,
+ attn_precision=attn_precision,
+ skip_reshape=skip_reshape,
+ skip_output_reshape=skip_output_reshape,
+ **kwargs
+ )
+
+ if skip_reshape:
+ B, H, L, D = q.shape
+ if H != heads:
+ return attention_pytorch(
+ q, k, v, heads,
+ mask=mask,
+ attn_precision=attn_precision,
+ skip_reshape=True,
+ skip_output_reshape=skip_output_reshape,
+ **kwargs
+ )
+ q_s, k_s, v_s = q, k, v
+ N = q.shape[2]
+ dim_head = D
+ else:
+ B, N, inner_dim = q.shape
+ if inner_dim % heads != 0:
+ return attention_pytorch(
+ q, k, v, heads,
+ mask=mask,
+ attn_precision=attn_precision,
+ skip_reshape=False,
+ skip_output_reshape=skip_output_reshape,
+ **kwargs
+ )
+ dim_head = inner_dim // heads
+
+ if dim_head >= 256 or N <= 1024:
+ return attention_pytorch(
+ q, k, v, heads,
+ mask=mask,
+ attn_precision=attn_precision,
+ skip_reshape=skip_reshape,
+ skip_output_reshape=skip_output_reshape,
+ **kwargs
+ )
+
+ if not skip_reshape:
+ q_s, k_s, v_s = map(
+ lambda t: t.view(B, -1, heads, dim_head).permute(0, 2, 1, 3).contiguous(),
+ (q, k, v),
+ )
+ B, H, L, D = q_s.shape
+
+ try:
+ out = sageattn3_blackwell(q_s, k_s, v_s, is_causal=False)
+ except Exception as e:
+ exception_fallback = True
+ logging.error("Error running SageAttention3: %s, falling back to pytorch attention.", e)
+
+ if exception_fallback:
+ if not skip_reshape:
+ del q_s, k_s, v_s
+ return attention_pytorch(
+ q, k, v, heads,
+ mask=mask,
+ attn_precision=attn_precision,
+ skip_reshape=False,
+ skip_output_reshape=skip_output_reshape,
+ **kwargs
+ )
+
+ if skip_reshape:
+ if not skip_output_reshape:
+ out = out.permute(0, 2, 1, 3).reshape(B, L, H * D)
+ else:
+ if skip_output_reshape:
+ pass
+ else:
+ out = out.permute(0, 2, 1, 3).reshape(B, L, H * D)
+
+ return out
try:
@torch.library.custom_op("flash_attention::flash_attn", mutates_args=())
@@ -650,6 +744,8 @@ optimized_attention_masked = optimized_attention
# register core-supported attention functions
if SAGE_ATTENTION_IS_AVAILABLE:
register_attention_function("sage", attention_sage)
+if SAGE_ATTENTION3_IS_AVAILABLE:
+ register_attention_function("sage3", attention3_sage)
if FLASH_ATTENTION_IS_AVAILABLE:
register_attention_function("flash", attention_flash)
if model_management.xformers_enabled():