Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Add some command line arguments to store text encoder weights in fp8.

comfyanonymous0cf4e8693945
diff --git a/comfy/cli_args.py b/comfy/cli_args.py
index e79b89c0f..72fce1087 100644
--- a/comfy/cli_args.py
+++ b/comfy/cli_args.py
@@ -62,6 +62,13 @@ fpvae_group.add_argument("--fp16-vae", action="store_true", help="Run the VAE in
fpvae_group.add_argument("--fp32-vae", action="store_true", help="Run the VAE in full precision fp32.")
fpvae_group.add_argument("--bf16-vae", action="store_true", help="Run the VAE in bf16.")
+fpte_group = parser.add_mutually_exclusive_group()
+fpte_group.add_argument("--fp8_e4m3fn-text-enc", action="store_true", help="Store text encoder weights in fp8 (e4m3fn variant).")
+fpte_group.add_argument("--fp8_e5m2-text-enc", action="store_true", help="Store text encoder weights in fp8 (e5m2 variant).")
+fpte_group.add_argument("--fp16-text-enc", action="store_true", help="Store text encoder weights in fp16.")
+fpte_group.add_argument("--fp32-text-enc", action="store_true", help="Store text encoder weights in fp32.")
+
+
parser.add_argument("--directml", type=int, nargs="?", metavar="DIRECTML_DEVICE", const=-1, help="Use torch-directml.")
parser.add_argument("--disable-ipex-optimize", action="store_true", help="Disables ipex.optimize when loading models with Intel GPUs.")
diff --git a/comfy/model_management.py b/comfy/model_management.py
index be4301aa4..d4acd8950 100644
--- a/comfy/model_management.py
+++ b/comfy/model_management.py
@@ -482,6 +482,21 @@ def text_encoder_device():
else:
return torch.device("cpu")
+def text_encoder_dtype(device=None):
+ if args.fp8_e4m3fn_text_enc:
+ return torch.float8_e4m3fn
+ elif args.fp8_e5m2_text_enc:
+ return torch.float8_e5m2
+ elif args.fp16_text_enc:
+ return torch.float16
+ elif args.fp32_text_enc:
+ return torch.float32
+
+ if should_use_fp16(device, prioritize_performance=False):
+ return torch.float16
+ else:
+ return torch.float32
+
def vae_device():
return get_torch_device()
diff --git a/comfy/sd.py b/comfy/sd.py
index 65d94f46e..c3cc8e720 100644
--- a/comfy/sd.py
+++ b/comfy/sd.py
@@ -95,10 +95,7 @@ class CLIP:
load_device = model_management.text_encoder_device()
offload_device = model_management.text_encoder_offload_device()
params['device'] = offload_device
- if model_management.should_use_fp16(load_device, prioritize_performance=False):
- params['dtype'] = torch.float16
- else:
- params['dtype'] = torch.float32
+ params['dtype'] = model_management.text_encoder_dtype(load_device)
self.cond_stage_model = clip(**(params))