Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Port some optimizations to flux model family. (#16488)

comfyanonymousb5cc8830279e
diff --git a/comfy/ldm/chroma/layers.py b/comfy/ldm/chroma/layers.py
index df348a8ed..1acbf0338 100644
--- a/comfy/ldm/chroma/layers.py
+++ b/comfy/ldm/chroma/layers.py
@@ -4,6 +4,7 @@ from torch import Tensor, nn
from comfy.ldm.flux.layers import (
MLPEmbedder,
ModulationOut,
+ modulated_norm,
)
# TODO: remove this in a few months
@@ -57,6 +58,6 @@ class LastLayer(nn.Module):
shift, scale = vec
shift = shift.squeeze(1)
scale = scale.squeeze(1)
- x = torch.addcmul(shift[:, None, :], 1 + scale[:, None, :], self.norm_final(x))
+ x = modulated_norm(x, self.norm_final, scale[:, None, :], shift[:, None, :])
x = self.linear(x)
return x
diff --git a/comfy/ldm/flux/layers.py b/comfy/ldm/flux/layers.py
index e28d704b4..1458236a6 100644
--- a/comfy/ldm/flux/layers.py
+++ b/comfy/ldm/flux/layers.py
@@ -4,6 +4,9 @@ from dataclasses import dataclass
import torch
from torch import Tensor, nn
+import comfy.model_management
+import comfy.quant_ops
+from comfy.ldm.modules.attention import AttentionTensorContainer, ComfyAttention
from .math import attention, rope
# Fix import for some custom nodes, TODO: delete eventually.
@@ -150,6 +153,12 @@ def apply_mod(tensor, m_mult, m_add=None, modulation_dims=None):
return tensor
+def modulated_norm(x, norm, scale, shift, modulation_dims=None):
+ if modulation_dims is not None or comfy.model_management.in_training:
+ return apply_mod(norm(x), 1 + scale, shift, modulation_dims)
+ return comfy.quant_ops.ck.adaln(x, scale, shift, norm.eps)
+
+
class SiLUActivation(nn.Module):
def __init__(self):
super().__init__()
@@ -163,6 +172,7 @@ class SiLUActivation(nn.Module):
class DoubleStreamBlock(nn.Module):
def __init__(self, hidden_size: int, num_heads: int, mlp_ratio: float, qkv_bias: bool = False, modulation=True, mlp_silu_act=False, proj_bias=True, yak_mlp=False, dtype=None, device=None, operations=None):
super().__init__()
+ self.comfy_attention = ComfyAttention()
mlp_hidden_dim = int(hidden_size * mlp_ratio)
self.num_heads = num_heads
@@ -200,8 +210,7 @@ class DoubleStreamBlock(nn.Module):
extra_options = transformer_options.copy()
# prepare image for attention
- img_modulated = self.img_norm1(img)
- img_modulated = apply_mod(img_modulated, (1 + img_mod1.scale), img_mod1.shift, modulation_dims_img)
+ img_modulated = modulated_norm(img, self.img_norm1, img_mod1.scale, img_mod1.shift, modulation_dims_img)
img_qkv = self.img_attn.qkv(img_modulated)
del img_modulated
img_q, img_k, img_v = img_qkv.view(img_qkv.shape[0], img_qkv.shape[1], 3, self.num_heads, -1).permute(2, 0, 3, 1, 4)
@@ -209,8 +218,7 @@ class DoubleStreamBlock(nn.Module):
img_q, img_k = self.img_attn.norm(img_q, img_k, img_v)
# prepare txt for attention
- txt_modulated = self.txt_norm1(txt)
- txt_modulated = apply_mod(txt_modulated, (1 + txt_mod1.scale), txt_mod1.shift, modulation_dims_txt)
+ txt_modulated = modulated_norm(txt, self.txt_norm1, txt_mod1.scale, txt_mod1.shift, modulation_dims_txt)
txt_qkv = self.txt_attn.qkv(txt_modulated)
del txt_modulated
txt_q, txt_k, txt_v = txt_qkv.view(txt_qkv.shape[0], txt_qkv.shape[1], 3, self.num_heads, -1).permute(2, 0, 3, 1, 4)
@@ -232,7 +240,8 @@ class DoubleStreamBlock(nn.Module):
q, k, v, pe, attn_mask = out.get("q", q), out.get("k", k), out.get("v", v), out.get("pe", pe), out.get("attn_mask", attn_mask)
# run actual attention
- attn = attention(q, k, v, pe=pe, mask=attn_mask, transformer_options=transformer_options)
+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)
+ attn = attention(q, k, v, pe=pe, mask=attn_mask, transformer_options=transformer_options, preferred_attention=self.comfy_attention)
del q, k, v
if "attn1_output_patch" in transformer_patches:
@@ -245,12 +254,12 @@ class DoubleStreamBlock(nn.Module):
# calculate the img bloks
img += apply_mod(self.img_attn.proj(img_attn), img_mod1.gate, None, modulation_dims_img)
del img_attn
- img += apply_mod(self.img_mlp(apply_mod(self.img_norm2(img), (1 + img_mod2.scale), img_mod2.shift, modulation_dims_img)), img_mod2.gate, None, modulation_dims_img)
+ img += apply_mod(self.img_mlp(modulated_norm(img, self.img_norm2, img_mod2.scale, img_mod2.shift, modulation_dims_img)), img_mod2.gate, None, modulation_dims_img)
# calculate the txt bloks
txt += apply_mod(self.txt_attn.proj(txt_attn), txt_mod1.gate, None, modulation_dims_txt)
del txt_attn
- txt += apply_mod(self.txt_mlp(apply_mod(self.txt_norm2(txt), (1 + txt_mod2.scale), txt_mod2.shift, modulation_dims_txt)), txt_mod2.gate, None, modulation_dims_txt)
+ txt += apply_mod(self.txt_mlp(modulated_norm(txt, self.txt_norm2, txt_mod2.scale, txt_mod2.shift, modulation_dims_txt)), txt_mod2.gate, None, modulation_dims_txt)
if txt.dtype == torch.float16:
txt = torch.nan_to_num(txt, nan=0.0, posinf=65504, neginf=-65504)
@@ -279,6 +288,7 @@ class SingleStreamBlock(nn.Module):
operations=None
):
super().__init__()
+ self.comfy_attention = ComfyAttention()
self.hidden_dim = hidden_size
self.num_heads = num_heads
head_dim = hidden_size // num_heads
@@ -322,7 +332,7 @@ class SingleStreamBlock(nn.Module):
transformer_patches = transformer_options.get("patches", {})
extra_options = transformer_options.copy()
- qkv, mlp = torch.split(self.linear1(apply_mod(self.pre_norm(x), (1 + mod.scale), mod.shift, modulation_dims)), [3 * self.hidden_size, self.mlp_hidden_dim_first], dim=-1)
+ qkv, mlp = torch.split(self.linear1(modulated_norm(x, self.pre_norm, mod.scale, mod.shift, modulation_dims)), [3 * self.hidden_size, self.mlp_hidden_dim_first], dim=-1)
q, k, v = qkv.view(qkv.shape[0], qkv.shape[1], 3, self.num_heads, -1).permute(2, 0, 3, 1, 4)
del qkv
@@ -335,7 +345,8 @@ class SingleStreamBlock(nn.Module):
q, k, v, pe, attn_mask = out.get("q", q), out.get("k", k), out.get("v", v), out.get("pe", pe), out.get("attn_mask", attn_mask)
# compute attention
- attn = attention(q, k, v, pe=pe, mask=attn_mask, transformer_options=transformer_options)
+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)
+ attn = attention(q, k, v, pe=pe, mask=attn_mask, transformer_options=transformer_options, preferred_attention=self.comfy_attention)
del q, k, v
if "attn1_output_patch" in transformer_patches:
@@ -367,6 +378,6 @@ class LastLayer(nn.Module):
vec = vec[:, None, :]
shift, scale = self.adaLN_modulation(vec).chunk(2, dim=-1)
- x = apply_mod(self.norm_final(x), (1 + scale), shift, modulation_dims)
+ x = modulated_norm(x, self.norm_final, scale, shift, modulation_dims)
x = self.linear(x)
return x
diff --git a/comfy/ldm/flux/math.py b/comfy/ldm/flux/math.py
index 891dea7dd..f5ac1bc5f 100644
--- a/comfy/ldm/flux/math.py
+++ b/comfy/ldm/flux/math.py
@@ -2,16 +2,19 @@ import torch
from einops import rearrange
from torch import Tensor
-from comfy.ldm.modules.attention import optimized_attention
+from comfy.ldm.modules.attention import AttentionTensorContainer, optimized_attention
import comfy.model_management
import comfy.quant_ops
-def attention(q: Tensor, k: Tensor, v: Tensor, pe: Tensor, mask=None, transformer_options={}) -> Tensor:
+def attention(q: Tensor, k: Tensor, v: Tensor, pe: Tensor, mask=None, transformer_options={}, preferred_attention=None) -> Tensor:
+ if isinstance(q, AttentionTensorContainer):
+ q, k, v = q.take(), k.take(), v.take()
if pe is not None:
q, k = apply_rope(q, k, pe)
heads = q.shape[1]
- x = optimized_attention(q, k, v, heads, skip_reshape=True, mask=mask, transformer_options=transformer_options)
+ q, k, v = AttentionTensorContainer(q), AttentionTensorContainer(k), AttentionTensorContainer(v)
+ x = optimized_attention(q, k, v, heads, skip_reshape=True, mask=mask, transformer_options=transformer_options, preferred_attention=preferred_attention)
return x
def rope(pos: Tensor, dim: int, theta: int) -> Tensor: