The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Automatically use fp8 for diffusion model weights if:
comfyanonymousba9095e5bd79
diff --git a/comfy/model_base.py b/comfy/model_base.pyindex ec15e9fcf..94f4d333c 100644--- a/comfy/model_base.py+++ b/comfy/model_base.py@@ -94,6 +94,7 @@ class BaseModel(torch.nn.Module):self.concat_keys = ()logging.info("model_type {}".format(model_type.name))logging.debug("adm {}".format(self.adm_channels))+ logging.info("model weight dtype {}, manual cast: {}".format(self.get_dtype(), self.manual_cast_dtype))self.memory_usage_factor = model_config.memory_usage_factordef apply_model(self, x, t, c_concat=None, c_crossattn=None, control=None, transformer_options={}, **kwargs):diff --git a/comfy/model_management.py b/comfy/model_management.pyindex da0b989a8..c0fb15095 100644--- a/comfy/model_management.py+++ b/comfy/model_management.py@@ -527,6 +527,9 @@ def unet_inital_load_device(parameters, dtype):else:return cpu_dev+def maximum_vram_for_weights(device=None):+ return (get_total_memory(device) * 0.8 - minimum_inference_memory())+def unet_dtype(device=None, model_params=0, supported_dtypes=[torch.float16, torch.bfloat16, torch.float32]):if args.bf16_unet:return torch.bfloat16@@ -536,6 +539,21 @@ def unet_dtype(device=None, model_params=0, supported_dtypes=[torch.float16, torreturn torch.float8_e4m3fnif args.fp8_e5m2_unet:return torch.float8_e5m2++ fp8_dtype = None+ try:+ for dtype in [torch.float8_e4m3fn, torch.float8_e5m2]:+ if dtype in supported_dtypes:+ fp8_dtype = dtype+ break+ except:+ pass++ if fp8_dtype is not None:+ free_model_memory = maximum_vram_for_weights(device)+ if model_params * 2 > free_model_memory:+ return fp8_dtype+if should_use_fp16(device=device, model_params=model_params, manual_cast=True):if torch.float16 in supported_dtypes:return torch.float16@@ -871,7 +889,7 @@ def should_use_fp16(device=None, model_params=0, prioritize_performance=True, mafp16_works = Trueif fp16_works or manual_cast:- free_model_memory = (get_free_memory() * 0.9 - minimum_inference_memory())+ free_model_memory = maximum_vram_for_weights(device)if (not prioritize_performance) or model_params * 4 > free_model_memory:return True@@ -920,7 +938,7 @@ def should_use_bf16(device=None, model_params=0, prioritize_performance=True, mabf16_works = torch.cuda.is_bf16_supported()if bf16_works or manual_cast:- free_model_memory = (get_free_memory() * 0.9 - minimum_inference_memory())+ free_model_memory = maximum_vram_for_weights(device)if (not prioritize_performance) or model_params * 4 > free_model_memory:return Truediff --git a/comfy/sd.py b/comfy/sd.pyindex 41ce18c80..bf336c859 100644--- a/comfy/sd.py+++ b/comfy/sd.py@@ -510,13 +510,14 @@ def load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=True, odiffusion_model_prefix = model_detection.unet_prefix_from_state_dict(sd)parameters = comfy.utils.calculate_parameters(sd, diffusion_model_prefix)+ weight_dtype = comfy.utils.weight_dtype(sd, diffusion_model_prefix)load_device = model_management.get_torch_device()model_config = model_detection.model_config_from_unet(sd, diffusion_model_prefix)if model_config is None:raise RuntimeError("ERROR: Could not detect model type of: {}".format(ckpt_path))- unet_dtype = model_management.unet_dtype(model_params=parameters, supported_dtypes=model_config.supported_inference_dtypes)+ unet_dtype = model_management.unet_dtype(model_params=parameters, supported_dtypes=[weight_dtype] + model_config.supported_inference_dtypes)manual_cast_dtype = model_management.unet_manual_cast(unet_dtype, load_device, model_config.supported_inference_dtypes)model_config.set_inference_dtype(unet_dtype, manual_cast_dtype)diff --git a/comfy/utils.py b/comfy/utils.pyindex 0db9fbb62..d9fe36f91 100644--- a/comfy/utils.py+++ b/comfy/utils.py@@ -40,9 +40,19 @@ def calculate_parameters(sd, prefix=""):params = 0for k in sd.keys():if k.startswith(prefix):- params += sd[k].nelement()+ w = sd[k]+ params += w.nelement()return params+def weight_dtype(sd, prefix=""):+ dtypes = {}+ for k in sd.keys():+ if k.startswith(prefix):+ w = sd[k]+ dtypes[w.dtype] = dtypes.get(w.dtype, 0) + 1++ return max(dtypes, key=dtypes.get)+def state_dict_key_replace(state_dict, keys_to_replace):for x in keys_to_replace:if x in state_dict: