The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Greatly improve lowvram sampling speed by getting rid of accelerate.
comfyanonymous36a7953142cc
diff --git a/comfy/controlnet.py b/comfy/controlnet.pyindex 110b5c7c2..8404054f3 100644--- a/comfy/controlnet.py+++ b/comfy/controlnet.py@@ -283,7 +283,7 @@ class ControlLora(ControlNet):cm = self.control_model.state_dict()for k in sd:- weight = comfy.model_management.resolve_lowvram_weight(sd[k], diffusion_model, k)+ weight = sd[k]try:comfy.utils.set_attr(self.control_model, k, weight)except:diff --git a/comfy/model_base.py b/comfy/model_base.pyindex f2a6f9841..b3a1fcd51 100644--- a/comfy/model_base.py+++ b/comfy/model_base.py@@ -162,11 +162,7 @@ class BaseModel(torch.nn.Module):def state_dict_for_saving(self, clip_state_dict, vae_state_dict):clip_state_dict = self.model_config.process_clip_state_dict_for_saving(clip_state_dict)- unet_sd = self.diffusion_model.state_dict()- unet_state_dict = {}- for k in unet_sd:- unet_state_dict[k] = comfy.model_management.resolve_lowvram_weight(unet_sd[k], self.diffusion_model, k)-+ unet_state_dict = self.diffusion_model.state_dict()unet_state_dict = self.model_config.process_unet_state_dict_for_saving(unet_state_dict)vae_state_dict = self.model_config.process_vae_state_dict_for_saving(vae_state_dict)if self.get_dtype() == torch.float16:diff --git a/comfy/model_management.py b/comfy/model_management.pyindex 23f39c985..61c967f64 100644--- a/comfy/model_management.py+++ b/comfy/model_management.py@@ -218,15 +218,8 @@ if args.force_fp16:FORCE_FP16 = Trueif lowvram_available:- try:- import accelerate- if set_vram_to in (VRAMState.LOW_VRAM, VRAMState.NO_VRAM):- vram_state = set_vram_to- except Exception as e:- import traceback- print(traceback.format_exc())- print("ERROR: LOW VRAM MODE NEEDS accelerate.")- lowvram_available = False+ if set_vram_to in (VRAMState.LOW_VRAM, VRAMState.NO_VRAM):+ vram_state = set_vram_toif cpu_state != CPUState.GPU:@@ -298,8 +291,20 @@ class LoadedModel:if lowvram_model_memory > 0:print("loading in lowvram mode", lowvram_model_memory/(1024 * 1024))- device_map = accelerate.infer_auto_device_map(self.real_model, max_memory={0: "{}MiB".format(lowvram_model_memory // (1024 * 1024)), "cpu": "16GiB"})- accelerate.dispatch_model(self.real_model, device_map=device_map, main_device=self.device)+ mem_counter = 0+ for m in self.real_model.modules():+ if hasattr(m, "comfy_cast_weights"):+ m.prev_comfy_cast_weights = m.comfy_cast_weights+ m.comfy_cast_weights = True+ module_mem = 0+ sd = m.state_dict()+ for k in sd:+ t = sd[k]+ module_mem += t.nelement() * t.element_size()+ if mem_counter + module_mem < lowvram_model_memory:+ m.to(self.device)+ mem_counter += module_mem+self.model_accelerated = Trueif is_intel_xpu() and not args.disable_ipex_optimize:@@ -309,7 +314,11 @@ class LoadedModel:def model_unload(self):if self.model_accelerated:- accelerate.hooks.remove_hook_from_submodules(self.real_model)+ for m in self.real_model.modules():+ if hasattr(m, "prev_comfy_cast_weights"):+ m.comfy_cast_weights = m.prev_comfy_cast_weights+ del m.prev_comfy_cast_weights+self.model_accelerated = Falseself.model.unpatch_model(self.model.offload_device)@@ -402,14 +411,14 @@ def load_models_gpu(models, memory_required=0):if lowvram_available and (vram_set_state == VRAMState.LOW_VRAM or vram_set_state == VRAMState.NORMAL_VRAM):model_size = loaded_model.model_memory_required(torch_dev)current_free_mem = get_free_memory(torch_dev)- lowvram_model_memory = int(max(256 * (1024 * 1024), (current_free_mem - 1024 * (1024 * 1024)) / 1.3 ))+ lowvram_model_memory = int(max(64 * (1024 * 1024), (current_free_mem - 1024 * (1024 * 1024)) / 1.3 ))if model_size > (current_free_mem - inference_memory): #only switch to lowvram if really necessaryvram_set_state = VRAMState.LOW_VRAMelse:lowvram_model_memory = 0if vram_set_state == VRAMState.NO_VRAM:- lowvram_model_memory = 256 * 1024 * 1024+ lowvram_model_memory = 64 * 1024 * 1024cur_loaded_model = loaded_model.model_load(lowvram_model_memory)current_loaded_models.insert(0, loaded_model)@@ -566,6 +575,11 @@ def supports_dtype(device, dtype): #TODOreturn Truereturn False+def device_supports_non_blocking(device):+ if is_device_mps(device):+ return False #pytorch bug? mps doesn't support non blocking+ return True+def cast_to_device(tensor, device, dtype, copy=False):device_supports_cast = Falseif tensor.dtype == torch.float32 or tensor.dtype == torch.float16:@@ -576,9 +590,7 @@ def cast_to_device(tensor, device, dtype, copy=False):elif is_intel_xpu():device_supports_cast = True- non_blocking = True- if is_device_mps(device):- non_blocking = False #pytorch bug? mps doesn't support non blocking+ non_blocking = device_supports_non_blocking(device)if device_supports_cast:if copy:@@ -742,11 +754,7 @@ def soft_empty_cache(force=False):torch.cuda.empty_cache()torch.cuda.ipc_collect()-def resolve_lowvram_weight(weight, model, key):- if weight.device == torch.device("meta"): #lowvram NOTE: this depends on the inner working of the accelerate library so it might break.- key_split = key.split('.') # I have no idea why they don't just leave the weight there instead of using the meta device.- op = comfy.utils.get_attr(model, '.'.join(key_split[:-1]))- weight = op._hf_hook.weights_map[key_split[-1]]+def resolve_lowvram_weight(weight, model, key): #TODO: removereturn weight#TODO: might be cleaner to put this somewhere elsediff --git a/comfy/ops.py b/comfy/ops.pyindex 08c633847..f6f85de60 100644--- a/comfy/ops.py+++ b/comfy/ops.py@@ -1,27 +1,93 @@import torchfrom contextlib import contextmanager+import comfy.model_management++def cast_bias_weight(s, input):+ bias = None+ non_blocking = comfy.model_management.device_supports_non_blocking(input.device)+ if s.bias is not None:+ bias = s.bias.to(device=input.device, dtype=input.dtype, non_blocking=non_blocking)+ weight = s.weight.to(device=input.device, dtype=input.dtype, non_blocking=non_blocking)+ return weight, bias+class disable_weight_init:class Linear(torch.nn.Linear):+ comfy_cast_weights = Falsedef reset_parameters(self):return None+ def forward_comfy_cast_weights(self, input):+ weight, bias = cast_bias_weight(self, input)+ return torch.nn.functional.linear(input, weight, bias)++ def forward(self, *args, **kwargs):+ if self.comfy_cast_weights:+ return self.forward_comfy_cast_weights(*args, **kwargs)+ else:+ return super().forward(*args, **kwargs)+class Conv2d(torch.nn.Conv2d):+ comfy_cast_weights = Falsedef reset_parameters(self):return None+ def forward_comfy_cast_weights(self, input):+ weight, bias = cast_bias_weight(self, input)+ return self._conv_forward(input, weight, bias)++ def forward(self, *args, **kwargs):+ if self.comfy_cast_weights:+ return self.forward_comfy_cast_weights(*args, **kwargs)+ else:+ return super().forward(*args, **kwargs)+class Conv3d(torch.nn.Conv3d):+ comfy_cast_weights = Falsedef reset_parameters(self):return None+ def forward_comfy_cast_weights(self, input):+ weight, bias = cast_bias_weight(self, input)+ return self._conv_forward(input, weight, bias)++ def forward(self, *args, **kwargs):+ if self.comfy_cast_weights:+ return self.forward_comfy_cast_weights(*args, **kwargs)+ else:+ return super().forward(*args, **kwargs)+class GroupNorm(torch.nn.GroupNorm):+ comfy_cast_weights = Falsedef reset_parameters(self):return None+ def forward_comfy_cast_weights(self, input):+ weight, bias = cast_bias_weight(self, input)+ return torch.nn.functional.group_norm(input, self.num_groups, weight, bias, self.eps)++ def forward(self, *args, **kwargs):+ if self.comfy_cast_weights:+ return self.forward_comfy_cast_weights(*args, **kwargs)+ else:+ return super().forward(*args, **kwargs)++class LayerNorm(torch.nn.LayerNorm):+ comfy_cast_weights = Falsedef reset_parameters(self):return None+ def forward_comfy_cast_weights(self, input):+ weight, bias = cast_bias_weight(self, input)+ return torch.nn.functional.layer_norm(input, self.normalized_shape, weight, bias, self.eps)++ def forward(self, *args, **kwargs):+ if self.comfy_cast_weights:+ return self.forward_comfy_cast_weights(*args, **kwargs)+ else:+ return super().forward(*args, **kwargs)+@classmethoddef conv_nd(s, dims, *args, **kwargs):if dims == 2:@@ -31,35 +97,19 @@ class disable_weight_init:else:raise ValueError(f"unsupported dimensions: {dims}")-def cast_bias_weight(s, input):- bias = None- if s.bias is not None:- bias = s.bias.to(device=input.device, dtype=input.dtype)- weight = s.weight.to(device=input.device, dtype=input.dtype)- return weight, biasclass manual_cast(disable_weight_init):class Linear(disable_weight_init.Linear):- def forward(self, input):- weight, bias = cast_bias_weight(self, input)- return torch.nn.functional.linear(input, weight, bias)+ comfy_cast_weights = Trueclass Conv2d(disable_weight_init.Conv2d):- def forward(self, input):- weight, bias = cast_bias_weight(self, input)- return self._conv_forward(input, weight, bias)+ comfy_cast_weights = Trueclass Conv3d(disable_weight_init.Conv3d):- def forward(self, input):- weight, bias = cast_bias_weight(self, input)- return self._conv_forward(input, weight, bias)+ comfy_cast_weights = Trueclass GroupNorm(disable_weight_init.GroupNorm):- def forward(self, input):- weight, bias = cast_bias_weight(self, input)- return torch.nn.functional.group_norm(input, self.num_groups, weight, bias, self.eps)+ comfy_cast_weights = Trueclass LayerNorm(disable_weight_init.LayerNorm):- def forward(self, input):- weight, bias = cast_bias_weight(self, input)- return torch.nn.functional.layer_norm(input, self.normalized_shape, weight, bias, self.eps)+ comfy_cast_weights = Truediff --git a/requirements.txt b/requirements.txtindex 14524485a..da1fbb27e 100644--- a/requirements.txt+++ b/requirements.txt@@ -4,7 +4,6 @@ einopstransformers>=4.25.1safetensors>=0.3.0aiohttp-acceleratepyyamlPillowscipy