The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Port some optimizations to flux model family. (#16488)
comfyanonymousb5cc8830279e
diff --git a/comfy/ldm/chroma/layers.py b/comfy/ldm/chroma/layers.pyindex df348a8ed..1acbf0338 100644--- a/comfy/ldm/chroma/layers.py+++ b/comfy/ldm/chroma/layers.py@@ -4,6 +4,7 @@ from torch import Tensor, nnfrom comfy.ldm.flux.layers import (MLPEmbedder,ModulationOut,+ modulated_norm,)# TODO: remove this in a few months@@ -57,6 +58,6 @@ class LastLayer(nn.Module):shift, scale = vecshift = shift.squeeze(1)scale = scale.squeeze(1)- x = torch.addcmul(shift[:, None, :], 1 + scale[:, None, :], self.norm_final(x))+ x = modulated_norm(x, self.norm_final, scale[:, None, :], shift[:, None, :])x = self.linear(x)return xdiff --git a/comfy/ldm/flux/layers.py b/comfy/ldm/flux/layers.pyindex e28d704b4..1458236a6 100644--- a/comfy/ldm/flux/layers.py+++ b/comfy/ldm/flux/layers.py@@ -4,6 +4,9 @@ from dataclasses import dataclassimport torchfrom torch import Tensor, nn+import comfy.model_management+import comfy.quant_ops+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttentionfrom .math import attention, rope# Fix import for some custom nodes, TODO: delete eventually.@@ -150,6 +153,12 @@ def apply_mod(tensor, m_mult, m_add=None, modulation_dims=None):return tensor+def modulated_norm(x, norm, scale, shift, modulation_dims=None):+ if modulation_dims is not None or comfy.model_management.in_training:+ return apply_mod(norm(x), 1 + scale, shift, modulation_dims)+ return comfy.quant_ops.ck.adaln(x, scale, shift, norm.eps)++class SiLUActivation(nn.Module):def __init__(self):super().__init__()@@ -163,6 +172,7 @@ class SiLUActivation(nn.Module):class DoubleStreamBlock(nn.Module):def __init__(self, hidden_size: int, num_heads: int, mlp_ratio: float, qkv_bias: bool = False, modulation=True, mlp_silu_act=False, proj_bias=True, yak_mlp=False, dtype=None, device=None, operations=None):super().__init__()+ self.comfy_attention = ComfyAttention()mlp_hidden_dim = int(hidden_size * mlp_ratio)self.num_heads = num_heads@@ -200,8 +210,7 @@ class DoubleStreamBlock(nn.Module):extra_options = transformer_options.copy()# prepare image for attention- img_modulated = self.img_norm1(img)- img_modulated = apply_mod(img_modulated, (1 + img_mod1.scale), img_mod1.shift, modulation_dims_img)+ img_modulated = modulated_norm(img, self.img_norm1, img_mod1.scale, img_mod1.shift, modulation_dims_img)img_qkv = self.img_attn.qkv(img_modulated)del img_modulatedimg_q, img_k, img_v = img_qkv.view(img_qkv.shape[0], img_qkv.shape[1], 3, self.num_heads, -1).permute(2, 0, 3, 1, 4)@@ -209,8 +218,7 @@ class DoubleStreamBlock(nn.Module):img_q, img_k = self.img_attn.norm(img_q, img_k, img_v)# prepare txt for attention- txt_modulated = self.txt_norm1(txt)- txt_modulated = apply_mod(txt_modulated, (1 + txt_mod1.scale), txt_mod1.shift, modulation_dims_txt)+ txt_modulated = modulated_norm(txt, self.txt_norm1, txt_mod1.scale, txt_mod1.shift, modulation_dims_txt)txt_qkv = self.txt_attn.qkv(txt_modulated)del txt_modulatedtxt_q, txt_k, txt_v = txt_qkv.view(txt_qkv.shape[0], txt_qkv.shape[1], 3, self.num_heads, -1).permute(2, 0, 3, 1, 4)@@ -232,7 +240,8 @@ class DoubleStreamBlock(nn.Module):q, k, v, pe, attn_mask = out.get("q", q), out.get("k", k), out.get("v", v), out.get("pe", pe), out.get("attn_mask", attn_mask)# run actual attention- attn = attention(q, k, v, pe=pe, mask=attn_mask, transformer_options=transformer_options)+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)+ attn = attention(q, k, v, pe=pe, mask=attn_mask, transformer_options=transformer_options, preferred_attention=self.comfy_attention)del q, k, vif "attn1_output_patch" in transformer_patches:@@ -245,12 +254,12 @@ class DoubleStreamBlock(nn.Module):# calculate the img bloksimg += apply_mod(self.img_attn.proj(img_attn), img_mod1.gate, None, modulation_dims_img)del img_attn- img += apply_mod(self.img_mlp(apply_mod(self.img_norm2(img), (1 + img_mod2.scale), img_mod2.shift, modulation_dims_img)), img_mod2.gate, None, modulation_dims_img)+ img += apply_mod(self.img_mlp(modulated_norm(img, self.img_norm2, img_mod2.scale, img_mod2.shift, modulation_dims_img)), img_mod2.gate, None, modulation_dims_img)# calculate the txt blokstxt += apply_mod(self.txt_attn.proj(txt_attn), txt_mod1.gate, None, modulation_dims_txt)del txt_attn- txt += apply_mod(self.txt_mlp(apply_mod(self.txt_norm2(txt), (1 + txt_mod2.scale), txt_mod2.shift, modulation_dims_txt)), txt_mod2.gate, None, modulation_dims_txt)+ txt += apply_mod(self.txt_mlp(modulated_norm(txt, self.txt_norm2, txt_mod2.scale, txt_mod2.shift, modulation_dims_txt)), txt_mod2.gate, None, modulation_dims_txt)if txt.dtype == torch.float16:txt = torch.nan_to_num(txt, nan=0.0, posinf=65504, neginf=-65504)@@ -279,6 +288,7 @@ class SingleStreamBlock(nn.Module):operations=None):super().__init__()+ self.comfy_attention = ComfyAttention()self.hidden_dim = hidden_sizeself.num_heads = num_headshead_dim = hidden_size // num_heads@@ -322,7 +332,7 @@ class SingleStreamBlock(nn.Module):transformer_patches = transformer_options.get("patches", {})extra_options = transformer_options.copy()- qkv, mlp = torch.split(self.linear1(apply_mod(self.pre_norm(x), (1 + mod.scale), mod.shift, modulation_dims)), [3 * self.hidden_size, self.mlp_hidden_dim_first], dim=-1)+ qkv, mlp = torch.split(self.linear1(modulated_norm(x, self.pre_norm, mod.scale, mod.shift, modulation_dims)), [3 * self.hidden_size, self.mlp_hidden_dim_first], dim=-1)q, k, v = qkv.view(qkv.shape[0], qkv.shape[1], 3, self.num_heads, -1).permute(2, 0, 3, 1, 4)del qkv@@ -335,7 +345,8 @@ class SingleStreamBlock(nn.Module):q, k, v, pe, attn_mask = out.get("q", q), out.get("k", k), out.get("v", v), out.get("pe", pe), out.get("attn_mask", attn_mask)# compute attention- attn = attention(q, k, v, pe=pe, mask=attn_mask, transformer_options=transformer_options)+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)+ attn = attention(q, k, v, pe=pe, mask=attn_mask, transformer_options=transformer_options, preferred_attention=self.comfy_attention)del q, k, vif "attn1_output_patch" in transformer_patches:@@ -367,6 +378,6 @@ class LastLayer(nn.Module):vec = vec[:, None, :]shift, scale = self.adaLN_modulation(vec).chunk(2, dim=-1)- x = apply_mod(self.norm_final(x), (1 + scale), shift, modulation_dims)+ x = modulated_norm(x, self.norm_final, scale, shift, modulation_dims)x = self.linear(x)return xdiff --git a/comfy/ldm/flux/math.py b/comfy/ldm/flux/math.pyindex 891dea7dd..f5ac1bc5f 100644--- a/comfy/ldm/flux/math.py+++ b/comfy/ldm/flux/math.py@@ -2,16 +2,19 @@ import torchfrom einops import rearrangefrom torch import Tensor-from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.modules.attention import AttentionTensorContainer, optimized_attentionimport comfy.model_managementimport comfy.quant_ops-def attention(q: Tensor, k: Tensor, v: Tensor, pe: Tensor, mask=None, transformer_options={}) -> Tensor:+def attention(q: Tensor, k: Tensor, v: Tensor, pe: Tensor, mask=None, transformer_options={}, preferred_attention=None) -> Tensor:+ if isinstance(q, AttentionTensorContainer):+ q, k, v = q.take(), k.take(), v.take()if pe is not None:q, k = apply_rope(q, k, pe)heads = q.shape[1]- x = optimized_attention(q, k, v, heads, skip_reshape=True, mask=mask, transformer_options=transformer_options)+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)+ x = optimized_attention(q, k, v, heads, skip_reshape=True, mask=mask, transformer_options=transformer_options, preferred_attention=preferred_attention)return xdef rope(pos: Tensor, dim: int, theta: int) -> Tensor: