The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Make scaled_dot_product switch to sliced attention on OOM.
comfyanonymousb8636a44aacd
diff --git a/comfy/ldm/modules/diffusionmodules/model.py b/comfy/ldm/modules/diffusionmodules/model.pyindex 05caf7312..91e7d60ec 100644--- a/comfy/ldm/modules/diffusionmodules/model.py+++ b/comfy/ldm/modules/diffusionmodules/model.py@@ -146,6 +146,41 @@ class ResnetBlock(nn.Module):return x+h+def slice_attention(q, k, v):+ r1 = torch.zeros_like(k, device=q.device)+ scale = (int(q.shape[-1])**(-0.5))++ mem_free_total = model_management.get_free_memory(q.device)++ gb = 1024 ** 3+ tensor_size = q.shape[0] * q.shape[1] * k.shape[2] * q.element_size()+ modifier = 3 if q.element_size() == 2 else 2.5+ mem_required = tensor_size * modifier+ steps = 1++ if mem_required > mem_free_total:+ steps = 2**(math.ceil(math.log(mem_required / mem_free_total, 2)))++ while True:+ try:+ slice_size = q.shape[1] // steps if (q.shape[1] % steps) == 0 else q.shape[1]+ for i in range(0, q.shape[1], slice_size):+ end = i + slice_size+ s1 = torch.bmm(q[:, i:end], k) * scale++ s2 = torch.nn.functional.softmax(s1, dim=2).permute(0,2,1)+ del s1++ r1[:, :, i:end] = torch.bmm(v, s2)+ del s2+ break+ except model_management.OOM_EXCEPTION as e:+ steps *= 2+ if steps > 128:+ raise e+ print("out of memory error, increasing steps and trying again", steps)++ return r1class AttnBlock(nn.Module):def __init__(self, in_channels):@@ -183,48 +218,15 @@ class AttnBlock(nn.Module):# compute attentionb,c,h,w = q.shape- scale = (int(c)**(-0.5))q = q.reshape(b,c,h*w)q = q.permute(0,2,1) # b,hw,ck = k.reshape(b,c,h*w) # b,c,hwv = v.reshape(b,c,h*w)- r1 = torch.zeros_like(k, device=q.device)-- mem_free_total = model_management.get_free_memory(q.device)-- gb = 1024 ** 3- tensor_size = q.shape[0] * q.shape[1] * k.shape[2] * q.element_size()- modifier = 3 if q.element_size() == 2 else 2.5- mem_required = tensor_size * modifier- steps = 1-- if mem_required > mem_free_total:- steps = 2**(math.ceil(math.log(mem_required / mem_free_total, 2)))-- while True:- try:- slice_size = q.shape[1] // steps if (q.shape[1] % steps) == 0 else q.shape[1]- for i in range(0, q.shape[1], slice_size):- end = i + slice_size- s1 = torch.bmm(q[:, i:end], k) * scale-- s2 = torch.nn.functional.softmax(s1, dim=2).permute(0,2,1)- del s1-- r1[:, :, i:end] = torch.bmm(v, s2)- del s2- break- except model_management.OOM_EXCEPTION as e:- steps *= 2- if steps > 128:- raise e- print("out of memory error, increasing steps and trying again", steps)-+ r1 = slice_attention(q, k, v)h_ = r1.reshape(b,c,h,w)del r1-h_ = self.proj_out(h_)return x+h_@@ -335,9 +337,14 @@ class MemoryEfficientAttnBlockPytorch(nn.Module):lambda t: t.view(B, 1, C, -1).transpose(2, 3).contiguous(),(q, k, v),)- out = torch.nn.functional.scaled_dot_product_attention(q, k, v, attn_mask=None, dropout_p=0.0, is_causal=False)- out = out.transpose(2, 3).reshape(B, C, H, W)+ try:+ out = torch.nn.functional.scaled_dot_product_attention(q, k, v, attn_mask=None, dropout_p=0.0, is_causal=False)+ out = out.transpose(2, 3).reshape(B, C, H, W)+ except model_management.OOM_EXCEPTION as e:+ print("scaled_dot_product_attention OOMed: switched to slice attention")+ out = slice_attention(q.view(B, -1, C), k.view(B, -1, C).transpose(1, 2), v.view(B, -1, C).transpose(1, 2)).reshape(B, C, H, W)+out = self.proj_out(out)return x+out