The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Use faster manual cast for fp8 in unet.
comfyanonymousba07cb748e47
diff --git a/comfy/model_base.py b/comfy/model_base.pyindex 5bfcc391d..bab7b9b34 100644--- a/comfy/model_base.py+++ b/comfy/model_base.py@@ -4,6 +4,7 @@ from comfy.ldm.modules.encoders.noise_aug_modules import CLIPEmbeddingNoiseAugmefrom comfy.ldm.modules.diffusionmodules.openaimodel import Timestepimport comfy.model_managementimport comfy.conds+import comfy.opsfrom enum import Enumimport contextlibfrom . import utils@@ -41,9 +42,14 @@ class BaseModel(torch.nn.Module):unet_config = model_config.unet_configself.latent_format = model_config.latent_formatself.model_config = model_config+ self.manual_cast_dtype = model_config.manual_cast_dtypeif not unet_config.get("disable_unet_model_creation", False):- self.diffusion_model = UNetModel(**unet_config, device=device)+ if self.manual_cast_dtype is not None:+ operations = comfy.ops.manual_cast+ else:+ operations = comfy.ops+ self.diffusion_model = UNetModel(**unet_config, device=device, operations=operations)self.model_type = model_typeself.model_sampling = model_sampling(model_config, model_type)@@ -63,11 +69,8 @@ class BaseModel(torch.nn.Module):context = c_crossattndtype = self.get_dtype()- if comfy.model_management.supports_dtype(xc.device, dtype):- precision_scope = lambda a: contextlib.nullcontext(a)- else:- precision_scope = torch.autocast- dtype = torch.float32+ if self.manual_cast_dtype is not None:+ dtype = self.manual_cast_dtypexc = xc.to(dtype)t = self.model_sampling.timestep(t).float()@@ -79,9 +82,7 @@ class BaseModel(torch.nn.Module):extra = extra.to(dtype)extra_conds[o] = extra- with precision_scope(comfy.model_management.get_autocast_device(xc.device)):- model_output = self.diffusion_model(xc, t, context=context, control=control, transformer_options=transformer_options, **extra_conds).float()-+ model_output = self.diffusion_model(xc, t, context=context, control=control, transformer_options=transformer_options, **extra_conds).float()return self.model_sampling.calculate_denoised(sigma, model_output, x)def get_dtype(self):diff --git a/comfy/model_management.py b/comfy/model_management.pyindex a6c8fb352..fe0374a8b 100644--- a/comfy/model_management.py+++ b/comfy/model_management.py@@ -474,6 +474,20 @@ def unet_dtype(device=None, model_params=0):return torch.float16return torch.float32+# None means no manual cast+def unet_manual_cast(weight_dtype, inference_device):+ if weight_dtype == torch.float32:+ return None++ fp16_supported = comfy.model_management.should_use_fp16(inference_device, prioritize_performance=False)+ if fp16_supported and weight_dtype == torch.float16:+ return None++ if fp16_supported:+ return torch.float16+ else:+ return torch.float32+def text_encoder_offload_device():if args.gpu_only:return get_torch_device()@@ -538,7 +552,7 @@ def get_autocast_device(dev):def supports_dtype(device, dtype): #TODOif dtype == torch.float32:return True- if torch.device("cpu") == device:+ if is_device_cpu(device):return Falseif dtype == torch.float16:return Truediff --git a/comfy/ops.py b/comfy/ops.pyindex e48568409..a67bc809f 100644--- a/comfy/ops.py+++ b/comfy/ops.py@@ -62,6 +62,15 @@ class manual_cast:weight, bias = cast_bias_weight(self, input)return torch.nn.functional.layer_norm(input, self.normalized_shape, weight, bias, self.eps)+ @classmethod+ def conv_nd(s, dims, *args, **kwargs):+ if dims == 2:+ return s.Conv2d(*args, **kwargs)+ elif dims == 3:+ return s.Conv3d(*args, **kwargs)+ else:+ raise ValueError(f"unsupported dimensions: {dims}")+@contextmanagerdef use_comfy_ops(device=None, dtype=None): # Kind of an ugly hack but I can't think of a better wayold_torch_nn_linear = torch.nn.Lineardiff --git a/comfy/sd.py b/comfy/sd.pyindex 43e201d36..8c056e4ea 100644--- a/comfy/sd.py+++ b/comfy/sd.py@@ -433,11 +433,15 @@ def load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=True, oparameters = comfy.utils.calculate_parameters(sd, "model.diffusion_model.")unet_dtype = model_management.unet_dtype(model_params=parameters)+ load_device = model_management.get_torch_device()+ manual_cast_dtype = model_management.unet_manual_cast(unet_dtype, load_device)class WeightsLoader(torch.nn.Module):passmodel_config = model_detection.model_config_from_unet(sd, "model.diffusion_model.", unet_dtype)+ model_config.set_manual_cast(manual_cast_dtype)+if model_config is None:raise RuntimeError("ERROR: Could not detect model type of: {}".format(ckpt_path))@@ -470,7 +474,7 @@ def load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=True, oprint("left over keys:", left_over)if output_model:- model_patcher = comfy.model_patcher.ModelPatcher(model, load_device=model_management.get_torch_device(), offload_device=model_management.unet_offload_device(), current_device=inital_load_device)+ model_patcher = comfy.model_patcher.ModelPatcher(model, load_device=load_device, offload_device=model_management.unet_offload_device(), current_device=inital_load_device)if inital_load_device != torch.device("cpu"):print("loaded straight to GPU")model_management.load_model_gpu(model_patcher)@@ -481,6 +485,9 @@ def load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=True, odef load_unet_state_dict(sd): #load unet in diffusers formatparameters = comfy.utils.calculate_parameters(sd)unet_dtype = model_management.unet_dtype(model_params=parameters)+ load_device = model_management.get_torch_device()+ manual_cast_dtype = model_management.unet_manual_cast(unet_dtype, load_device)+if "input_blocks.0.0.weight" in sd: #ldmmodel_config = model_detection.model_config_from_unet(sd, "", unet_dtype)if model_config is None:@@ -501,13 +508,14 @@ def load_unet_state_dict(sd): #load unet in diffusers formatelse:print(diffusers_keys[k], k)offload_device = model_management.unet_offload_device()+ model_config.set_manual_cast(manual_cast_dtype)model = model_config.get_model(new_sd, "")model = model.to(offload_device)model.load_model_weights(new_sd, "")left_over = sd.keys()if len(left_over) > 0:print("left over keys in unet:", left_over)- return comfy.model_patcher.ModelPatcher(model, load_device=model_management.get_torch_device(), offload_device=offload_device)+ return comfy.model_patcher.ModelPatcher(model, load_device=load_device, offload_device=offload_device)def load_unet(unet_path):sd = comfy.utils.load_torch_file(unet_path)diff --git a/comfy/supported_models_base.py b/comfy/supported_models_base.pyindex 3412cfea0..49087d23e 100644--- a/comfy/supported_models_base.py+++ b/comfy/supported_models_base.py@@ -22,6 +22,8 @@ class BASE:sampling_settings = {}latent_format = latent_formats.LatentFormat+ manual_cast_dtype = None+@classmethoddef matches(s, unet_config):for k in s.unet_config:@@ -71,3 +73,5 @@ class BASE:replace_prefix = {"": "first_stage_model."}return utils.state_dict_prefix_replace(state_dict, replace_prefix)+ def set_manual_cast(self, manual_cast_dtype):+ self.manual_cast_dtype = manual_cast_dtype