The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Add support for sage attention 3 in comfyui, enable via new cli arg (#11026)
mengqin0357ed7ec4a1
diff --git a/comfy/ldm/modules/attention.py b/comfy/ldm/modules/attention.pyindex a8800ded0..ccf690945 100644--- a/comfy/ldm/modules/attention.py+++ b/comfy/ldm/modules/attention.py@@ -30,6 +30,13 @@ except ImportError as e:raise eexit(-1)+SAGE_ATTENTION3_IS_AVAILABLE = False+try:+ from sageattn3 import sageattn3_blackwell+ SAGE_ATTENTION3_IS_AVAILABLE = True+except ImportError:+ pass+FLASH_ATTENTION_IS_AVAILABLE = Falsetry:from flash_attn import flash_attn_func@@ -563,6 +570,93 @@ def attention_sage(q, k, v, heads, mask=None, attn_precision=None, skip_reshape=out = out.reshape(b, -1, heads * dim_head)return out+@wrap_attn+def attention3_sage(q, k, v, heads, mask=None, attn_precision=None, skip_reshape=False, skip_output_reshape=False, **kwargs):+ exception_fallback = False+ if (q.device.type != "cuda" or+ q.dtype not in (torch.float16, torch.bfloat16) or+ mask is not None):+ return attention_pytorch(+ q, k, v, heads,+ mask=mask,+ attn_precision=attn_precision,+ skip_reshape=skip_reshape,+ skip_output_reshape=skip_output_reshape,+ **kwargs+ )++ if skip_reshape:+ B, H, L, D = q.shape+ if H != heads:+ return attention_pytorch(+ q, k, v, heads,+ mask=mask,+ attn_precision=attn_precision,+ skip_reshape=True,+ skip_output_reshape=skip_output_reshape,+ **kwargs+ )+ q_s, k_s, v_s = q, k, v+ N = q.shape[2]+ dim_head = D+ else:+ B, N, inner_dim = q.shape+ if inner_dim % heads != 0:+ return attention_pytorch(+ q, k, v, heads,+ mask=mask,+ attn_precision=attn_precision,+ skip_reshape=False,+ skip_output_reshape=skip_output_reshape,+ **kwargs+ )+ dim_head = inner_dim // heads++ if dim_head >= 256 or N <= 1024:+ return attention_pytorch(+ q, k, v, heads,+ mask=mask,+ attn_precision=attn_precision,+ skip_reshape=skip_reshape,+ skip_output_reshape=skip_output_reshape,+ **kwargs+ )++ if not skip_reshape:+ q_s, k_s, v_s = map(+ lambda t: t.view(B, -1, heads, dim_head).permute(0, 2, 1, 3).contiguous(),+ (q, k, v),+ )+ B, H, L, D = q_s.shape++ try:+ out = sageattn3_blackwell(q_s, k_s, v_s, is_causal=False)+ except Exception as e:+ exception_fallback = True+ logging.error("Error running SageAttention3: %s, falling back to pytorch attention.", e)++ if exception_fallback:+ if not skip_reshape:+ del q_s, k_s, v_s+ return attention_pytorch(+ q, k, v, heads,+ mask=mask,+ attn_precision=attn_precision,+ skip_reshape=False,+ skip_output_reshape=skip_output_reshape,+ **kwargs+ )++ if skip_reshape:+ if not skip_output_reshape:+ out = out.permute(0, 2, 1, 3).reshape(B, L, H * D)+ else:+ if skip_output_reshape:+ pass+ else:+ out = out.permute(0, 2, 1, 3).reshape(B, L, H * D)++ return outtry:@torch.library.custom_op("flash_attention::flash_attn", mutates_args=())@@ -650,6 +744,8 @@ optimized_attention_masked = optimized_attention# register core-supported attention functionsif SAGE_ATTENTION_IS_AVAILABLE:register_attention_function("sage", attention_sage)+if SAGE_ATTENTION3_IS_AVAILABLE:+ register_attention_function("sage3", attention3_sage)if FLASH_ATTENTION_IS_AVAILABLE:register_attention_function("flash", attention_flash)if model_management.xformers_enabled():