The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Add a weight_dtype fp8_e4m3fn_fast to the Diffusion Model Loader node.
comfyanonymouse38c94228bce
diff --git a/comfy/model_base.py b/comfy/model_base.pyindex 9bfdb3b3e..a98fee1d9 100644--- a/comfy/model_base.py+++ b/comfy/model_base.py@@ -96,7 +96,7 @@ class BaseModel(torch.nn.Module):if not unet_config.get("disable_unet_model_creation", False):if model_config.custom_operations is None:- operations = comfy.ops.pick_operations(unet_config.get("dtype", None), self.manual_cast_dtype)+ operations = comfy.ops.pick_operations(unet_config.get("dtype", None), self.manual_cast_dtype, fp8_optimizations=model_config.optimizations.get("fp8", False))else:operations = model_config.custom_operationsself.diffusion_model = unet_model(**unet_config, device=device, operations=operations)diff --git a/comfy/model_management.py b/comfy/model_management.pyindex a97d489d5..09798bd0f 100644--- a/comfy/model_management.py+++ b/comfy/model_management.py@@ -145,7 +145,7 @@ total_ram = psutil.virtual_memory().total / (1024 * 1024)logging.info("Total VRAM {:0.0f} MB, total RAM {:0.0f} MB".format(total_vram, total_ram))try:- logging.info("pytorch version: {}".format(torch.version.__version__))+ logging.info("pytorch version: {}".format(torch_version))except:pass@@ -1065,6 +1065,9 @@ def should_use_bf16(device=None, model_params=0, prioritize_performance=True, mareturn Falsedef supports_fp8_compute(device=None):+ if not is_nvidia():+ return False+props = torch.cuda.get_device_properties(device)if props.major >= 9:return True@@ -1072,6 +1075,14 @@ def supports_fp8_compute(device=None):return Falseif props.minor < 9:return False++ if int(torch_version[0]) < 2 or (int(torch_version[0]) == 2 and int(torch_version[2]) < 3):+ return False++ if WINDOWS:+ if (int(torch_version[0]) == 2 and int(torch_version[2]) < 4):+ return False+return Truedef soft_empty_cache(force=False):diff --git a/comfy/ops.py b/comfy/ops.pyindex f9411ba59..c90e25ead 100644--- a/comfy/ops.py+++ b/comfy/ops.py@@ -299,7 +299,11 @@ class fp8_ops(manual_cast):return torch.nn.functional.linear(input, weight, bias)-def pick_operations(weight_dtype, compute_dtype, load_device=None, disable_fast_fp8=False):+def pick_operations(weight_dtype, compute_dtype, load_device=None, disable_fast_fp8=False, fp8_optimizations=False):+ if comfy.model_management.supports_fp8_compute(load_device):+ if (fp8_optimizations or args.fast) and not disable_fast_fp8:+ return fp8_ops+if compute_dtype is None or weight_dtype == compute_dtype:return disable_weight_initif args.fast and not disable_fast_fp8:diff --git a/comfy/sd.py b/comfy/sd.pyindex a494e531c..feb1138d7 100644--- a/comfy/sd.py+++ b/comfy/sd.py@@ -433,6 +433,7 @@ def detect_te_model(sd):def load_text_encoder_state_dicts(state_dicts=[], embedding_directory=None, clip_type=CLIPType.STABLE_DIFFUSION, model_options={}):clip_data = state_dicts+class EmptyClass:pass@@ -592,7 +593,6 @@ def load_state_dict_guess_config(sd, output_vae=True, output_clip=True, output_cif output_model:inital_load_device = model_management.unet_inital_load_device(parameters, unet_dtype)- offload_device = model_management.unet_offload_device()model = model_config.get_model(sd, diffusion_model_prefix, device=inital_load_device)model.load_model_weights(sd, diffusion_model_prefix)@@ -678,6 +678,9 @@ def load_diffusion_model_state_dict(sd, model_options={}): #load unet in diffusemanual_cast_dtype = model_management.unet_manual_cast(unet_dtype, load_device, model_config.supported_inference_dtypes)model_config.set_inference_dtype(unet_dtype, manual_cast_dtype)model_config.custom_operations = model_options.get("custom_operations", model_config.custom_operations)+ if model_options.get("fp8_optimizations", False):+ model_config.optimizations["fp8"] = True+model = model_config.get_model(new_sd, "")model = model.to(offload_device)model.load_model_weights(new_sd, "")diff --git a/comfy/supported_models_base.py b/comfy/supported_models_base.pyindex 7a2152f91..68e895519 100644--- a/comfy/supported_models_base.py+++ b/comfy/supported_models_base.py@@ -49,6 +49,7 @@ class BASE:manual_cast_dtype = Nonecustom_operations = None+ optimizations = {"fp8": False}@classmethoddef matches(s, unet_config, state_dict=None):diff --git a/nodes.py b/nodes.pyindex a4065c764..15a783529 100644--- a/nodes.py+++ b/nodes.py@@ -861,7 +861,7 @@ class UNETLoader:@classmethoddef INPUT_TYPES(s):return {"required": { "unet_name": (folder_paths.get_filename_list("diffusion_models"), ),- "weight_dtype": (["default", "fp8_e4m3fn", "fp8_e5m2"],)+ "weight_dtype": (["default", "fp8_e4m3fn", "fp8_e4m3fn_fast", "fp8_e5m2"],)}}RETURN_TYPES = ("MODEL",)FUNCTION = "load_unet"@@ -872,6 +872,9 @@ class UNETLoader:model_options = {}if weight_dtype == "fp8_e4m3fn":model_options["dtype"] = torch.float8_e4m3fn+ elif weight_dtype == "fp8_e4m3fn_fast":+ model_options["dtype"] = torch.float8_e4m3fn+ model_options["fp8_optimizations"] = Trueelif weight_dtype == "fp8_e5m2":model_options["dtype"] = torch.float8_e5m2