The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Lowvram mode for gligen and fix some lowvram issues.
comfyanonymouscb1551b819ec
diff --git a/comfy/gligen.py b/comfy/gligen.pyindex 45b674503..8c7cb432e 100644--- a/comfy/gligen.py+++ b/comfy/gligen.py@@ -242,14 +242,28 @@ class Gligen(nn.Module):self.position_net = position_netself.key_dim = key_dimself.max_objs = 30+ self.lowvram = Falsedef _set_position(self, boxes, masks, positive_embeddings):+ if self.lowvram == True:+ self.position_net.to(boxes.device)+objs = self.position_net(boxes, masks, positive_embeddings)- def func(key, x):- module = self.module_list[key]- return module(x, objs)- return func+ if self.lowvram == True:+ self.position_net.cpu()+ def func_lowvram(key, x):+ module = self.module_list[key]+ module.to(x.device)+ r = module(x, objs)+ module.cpu()+ return r+ return func_lowvram+ else:+ def func(key, x):+ module = self.module_list[key]+ return module(x, objs)+ return funcdef set_position(self, latent_image_shape, position_params, device):batch, c, h, w = latent_image_shape@@ -294,8 +308,11 @@ class Gligen(nn.Module):masks.to(device),conds.to(device))+ def set_lowvram(self, value=True):+ self.lowvram = value+def cleanup(self):- pass+ self.lowvram = Falsedef get_models(self):return [self]diff --git a/comfy/ldm/modules/attention.py b/comfy/ldm/modules/attention.pyindex 5eabecd65..573f4e1c6 100644--- a/comfy/ldm/modules/attention.py+++ b/comfy/ldm/modules/attention.py@@ -572,9 +572,6 @@ class BasicTransformerBlock(nn.Module):x += nx = self.ff(self.norm3(x)) + x-- if current_index is not None:- transformer_options["current_index"] += 1return xdiff --git a/comfy/ldm/modules/diffusionmodules/openaimodel.py b/comfy/ldm/modules/diffusionmodules/openaimodel.pyindex 4352b756d..5aef23f33 100644--- a/comfy/ldm/modules/diffusionmodules/openaimodel.py+++ b/comfy/ldm/modules/diffusionmodules/openaimodel.py@@ -88,6 +88,19 @@ class TimestepEmbedSequential(nn.Sequential, TimestepBlock):x = layer(x)return x+#This is needed because accelerate makes a copy of transformer_options which breaks "current_index"+def forward_timestep_embed(ts, x, emb, context=None, transformer_options={}, output_shape=None):+ for layer in ts:+ if isinstance(layer, TimestepBlock):+ x = layer(x, emb)+ elif isinstance(layer, SpatialTransformer):+ x = layer(x, context, transformer_options)+ transformer_options["current_index"] += 1+ elif isinstance(layer, Upsample):+ x = layer(x, output_shape=output_shape)+ else:+ x = layer(x)+ return xclass Upsample(nn.Module):"""@@ -805,13 +818,13 @@ class UNetModel(nn.Module):h = x.type(self.dtype)for id, module in enumerate(self.input_blocks):- h = module(h, emb, context, transformer_options)+ h = forward_timestep_embed(module, h, emb, context, transformer_options)if control is not None and 'input' in control and len(control['input']) > 0:ctrl = control['input'].pop()if ctrl is not None:h += ctrlhs.append(h)- h = self.middle_block(h, emb, context, transformer_options)+ h = forward_timestep_embed(self.middle_block, h, emb, context, transformer_options)if control is not None and 'middle' in control and len(control['middle']) > 0:h += control['middle'].pop()@@ -828,7 +841,7 @@ class UNetModel(nn.Module):output_shape = hs[-1].shapeelse:output_shape = None- h = module(h, emb, context, transformer_options, output_shape)+ h = forward_timestep_embed(module, h, emb, context, transformer_options, output_shape)h = h.type(x.dtype)if self.predict_codebook_ids:return self.id_predictor(h)diff --git a/comfy/model_management.py b/comfy/model_management.pyindex 3aea7ea8e..7070912df 100644--- a/comfy/model_management.py+++ b/comfy/model_management.py@@ -201,6 +201,9 @@ def load_controlnet_gpu(control_models):returnif vram_state == VRAMState.LOW_VRAM or vram_state == VRAMState.NO_VRAM:+ for m in control_models:+ if hasattr(m, 'set_lowvram'):+ m.set_lowvram(True)#don't load controlnets like this if low vram because they will be loaded right before running and unloaded right afterreturn