The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
flux: reduce VRAM usage (#10737)
rattus94c298f9625b
diff --git a/comfy/ldm/flux/layers.py b/comfy/ldm/flux/layers.pyindex a3eab0470..f4bf56e01 100644--- a/comfy/ldm/flux/layers.py+++ b/comfy/ldm/flux/layers.py@@ -167,39 +167,55 @@ class DoubleStreamBlock(nn.Module):img_modulated = self.img_norm1(img)img_modulated = apply_mod(img_modulated, (1 + img_mod1.scale), img_mod1.shift, modulation_dims_img)img_qkv = self.img_attn.qkv(img_modulated)+ del img_modulatedimg_q, img_k, img_v = img_qkv.view(img_qkv.shape[0], img_qkv.shape[1], 3, self.num_heads, -1).permute(2, 0, 3, 1, 4)+ del img_qkvimg_q, img_k = self.img_attn.norm(img_q, img_k, img_v)# prepare txt for attentiontxt_modulated = self.txt_norm1(txt)txt_modulated = apply_mod(txt_modulated, (1 + txt_mod1.scale), txt_mod1.shift, modulation_dims_txt)txt_qkv = self.txt_attn.qkv(txt_modulated)+ del txt_modulatedtxt_q, txt_k, txt_v = txt_qkv.view(txt_qkv.shape[0], txt_qkv.shape[1], 3, self.num_heads, -1).permute(2, 0, 3, 1, 4)+ del txt_qkvtxt_q, txt_k = self.txt_attn.norm(txt_q, txt_k, txt_v)if self.flipped_img_txt:+ q = torch.cat((img_q, txt_q), dim=2)+ del img_q, txt_q+ k = torch.cat((img_k, txt_k), dim=2)+ del img_k, txt_k+ v = torch.cat((img_v, txt_v), dim=2)+ del img_v, txt_v# run actual attention- attn = attention(torch.cat((img_q, txt_q), dim=2),- torch.cat((img_k, txt_k), dim=2),- torch.cat((img_v, txt_v), dim=2),+ attn = attention(q, k, v,pe=pe, mask=attn_mask, transformer_options=transformer_options)+ del q, k, vimg_attn, txt_attn = attn[:, : img.shape[1]], attn[:, img.shape[1]:]else:+ q = torch.cat((txt_q, img_q), dim=2)+ del txt_q, img_q+ k = torch.cat((txt_k, img_k), dim=2)+ del txt_k, img_k+ v = torch.cat((txt_v, img_v), dim=2)+ del txt_v, img_v# run actual attention- attn = attention(torch.cat((txt_q, img_q), dim=2),- torch.cat((txt_k, img_k), dim=2),- torch.cat((txt_v, img_v), dim=2),+ attn = attention(q, k, v,pe=pe, mask=attn_mask, transformer_options=transformer_options)+ del q, k, vtxt_attn, img_attn = attn[:, : txt.shape[1]], attn[:, txt.shape[1]:]# calculate the img bloksimg += apply_mod(self.img_attn.proj(img_attn), img_mod1.gate, None, modulation_dims_img)+ del img_attnimg += apply_mod(self.img_mlp(apply_mod(self.img_norm2(img), (1 + img_mod2.scale), img_mod2.shift, modulation_dims_img)), img_mod2.gate, None, modulation_dims_img)# calculate the txt blokstxt += apply_mod(self.txt_attn.proj(txt_attn), txt_mod1.gate, None, modulation_dims_txt)+ del txt_attntxt += apply_mod(self.txt_mlp(apply_mod(self.txt_norm2(txt), (1 + txt_mod2.scale), txt_mod2.shift, modulation_dims_txt)), txt_mod2.gate, None, modulation_dims_txt)if txt.dtype == torch.float16:@@ -249,12 +265,15 @@ class SingleStreamBlock(nn.Module):qkv, mlp = torch.split(self.linear1(apply_mod(self.pre_norm(x), (1 + mod.scale), mod.shift, modulation_dims)), [3 * self.hidden_size, self.mlp_hidden_dim], dim=-1)q, k, v = qkv.view(qkv.shape[0], qkv.shape[1], 3, self.num_heads, -1).permute(2, 0, 3, 1, 4)+ del qkvq, k = self.norm(q, k, v)# compute attentionattn = attention(q, k, v, pe=pe, mask=attn_mask, transformer_options=transformer_options)+ del q, k, v# compute activation in mlp stream, cat again and run second linear layer- output = self.linear2(torch.cat((attn, self.mlp_act(mlp)), 2))+ mlp = self.mlp_act(mlp)+ output = self.linear2(torch.cat((attn, mlp), 2))x += apply_mod(output, mod.gate, None, modulation_dims)if x.dtype == torch.float16:x = torch.nan_to_num(x, nan=0.0, posinf=65504, neginf=-65504)