The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Speed up lora compute and lower memory usage by doing it in fp16. (#11161)
comfyanonymous50ca97e7765d
diff --git a/comfy/model_management.py b/comfy/model_management.pyindex aeddbaefe..40717b1e4 100644--- a/comfy/model_management.py+++ b/comfy/model_management.py@@ -1492,6 +1492,20 @@ def extended_fp16_support():return True+LORA_COMPUTE_DTYPES = {}+def lora_compute_dtype(device):+ dtype = LORA_COMPUTE_DTYPES.get(device, None)+ if dtype is not None:+ return dtype++ if should_use_fp16(device):+ dtype = torch.float16+ else:+ dtype = torch.float32++ LORA_COMPUTE_DTYPES[device] = dtype+ return dtype+def soft_empty_cache(force=False):global cpu_stateif cpu_state == CPUState.MPS:diff --git a/comfy/model_patcher.py b/comfy/model_patcher.pyindex 215784874..4f076a6aa 100644--- a/comfy/model_patcher.py+++ b/comfy/model_patcher.py@@ -614,10 +614,11 @@ class ModelPatcher:if key not in self.backup:self.backup[key] = collections.namedtuple('Dimension', ['weight', 'inplace_update'])(weight.to(device=self.offload_device, copy=inplace_update), inplace_update)+ temp_dtype = comfy.model_management.lora_compute_dtype(device_to)if device_to is not None:- temp_weight = comfy.model_management.cast_to_device(weight, device_to, torch.float32, copy=True)+ temp_weight = comfy.model_management.cast_to_device(weight, device_to, temp_dtype, copy=True)else:- temp_weight = weight.to(torch.float32, copy=True)+ temp_weight = weight.to(temp_dtype, copy=True)if convert_func is not None:temp_weight = convert_func(temp_weight, inplace=True)