The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Add CublasOps support (#7574)
catboxanon1714a4c158c1
diff --git a/comfy/cli_args.py b/comfy/cli_args.pyindex 79ecbd682..81f29f098 100644--- a/comfy/cli_args.py+++ b/comfy/cli_args.py@@ -136,8 +136,9 @@ parser.add_argument("--deterministic", action="store_true", help="Make pytorch uclass PerformanceFeature(enum.Enum):Fp16Accumulation = "fp16_accumulation"Fp8MatrixMultiplication = "fp8_matrix_mult"+ CublasOps = "cublas_ops"-parser.add_argument("--fast", nargs="*", type=PerformanceFeature, help="Enable some untested and potentially quality deteriorating optimizations. --fast with no arguments enables everything. You can pass a list specific optimizations if you only want to enable specific ones. Current valid optimizations: fp16_accumulation fp8_matrix_mult")+parser.add_argument("--fast", nargs="*", type=PerformanceFeature, help="Enable some untested and potentially quality deteriorating optimizations. --fast with no arguments enables everything. You can pass a list specific optimizations if you only want to enable specific ones. Current valid optimizations: fp16_accumulation fp8_matrix_mult cublas_ops")parser.add_argument("--dont-print-server", action="store_true", help="Don't print server output.")parser.add_argument("--quick-test-for-ci", action="store_true", help="Quick test for CI.")diff --git a/comfy/ops.py b/comfy/ops.pyindex ced461011..9a5c1ee99 100644--- a/comfy/ops.py+++ b/comfy/ops.py@@ -357,6 +357,25 @@ def scaled_fp8_ops(fp8_matrix_mult=False, scale_input=False, override_dtype=Nonereturn scaled_fp8_op+CUBLAS_IS_AVAILABLE = False+try:+ from cublas_ops import CublasLinear+ CUBLAS_IS_AVAILABLE = True+except ImportError:+ pass++if CUBLAS_IS_AVAILABLE:+ class cublas_ops(disable_weight_init):+ class Linear(CublasLinear, disable_weight_init.Linear):+ def reset_parameters(self):+ return None++ def forward_comfy_cast_weights(self, input):+ return super().forward(input)++ def forward(self, *args, **kwargs):+ return super().forward(*args, **kwargs)+def pick_operations(weight_dtype, compute_dtype, load_device=None, disable_fast_fp8=False, fp8_optimizations=False, scaled_fp8=None):fp8_compute = comfy.model_management.supports_fp8_compute(load_device)if scaled_fp8 is not None:@@ -369,6 +388,15 @@ def pick_operations(weight_dtype, compute_dtype, load_device=None, disable_fast_):return fp8_ops+ if (+ PerformanceFeature.CublasOps in args.fast and+ CUBLAS_IS_AVAILABLE and+ weight_dtype == torch.float16 and+ (compute_dtype == torch.float16 or compute_dtype is None)+ ):+ logging.info("Using cublas ops")+ return cublas_ops+if compute_dtype is None or weight_dtype == compute_dtype:return disable_weight_init