The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Cleaner way of enabling quantized mm on text gen but not on text enc. (#16189)
comfyanonymous249c5a3b9518
diff --git a/comfy/ops.py b/comfy/ops.pyindex d53886de0..24631840a 100644--- a/comfy/ops.py+++ b/comfy/ops.py@@ -1287,6 +1287,16 @@ def _quantized_weight_state_dict(module, sd, prefix, extra_quant_conf=None, extrreturn sd+class MixedPrecisionOp(CastWeightBiasOp):+ quant_format = None++ def can_use_quantized_matmul(self, disabled_formats):+ return (self.quant_format in QUANT_ALGOS+ and not self._full_precision_mm_config+ and self.quant_format not in self._disabled_formats+ and self.quant_format not in disabled_formats)++def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_precision_mm=False, disabled=[]):class MixedPrecisionOps(manual_cast):_quant_config = quant_config@@ -1294,7 +1304,7 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec_full_precision_mm = full_precision_mm_disabled = disabled- class Linear(torch.nn.Module, CastWeightBiasOp):+ class Linear(torch.nn.Module, MixedPrecisionOp):_disabled_formats = disableddef __init__(self, in_features: int, out_features: int, bias: bool = True, device=None, dtype=None):@@ -1342,8 +1352,7 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_preccompute_dtype=compute_dtype,want_requant=want_requant,) as (weight, bias):- if isinstance(weight, QuantizedTensor) and (self._full_precision_mm_config or getattr(self, "quant_format", None) in self._disabled_formats or not weight.layout_cls.supports_fast_matmul()):- # explicit per-layer full precision, or a format this device can't run: don't reach the fast quantized matmul+ if self._full_precision_mm and isinstance(weight, QuantizedTensor):weight = weight.dequantize()return self._forward(input, weight, bias)@@ -1447,7 +1456,7 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_precdef _apply(self, fn, recurse=True): # This is to get torch.compile + moving weights to another device workingreturn _quantized_apply(self, fn, recurse)- class MoEExperts(torch.nn.Module, CastWeightBiasOp):+ class MoEExperts(torch.nn.Module, MixedPrecisionOp):"""Container for E quantized expert weights, indexed via expert_weight(i).The bank lives on self.weight as a single 3D tensor — either a@@ -1651,29 +1660,45 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_precreturn MixedPrecisionOps-def pick_operations(weight_dtype, compute_dtype, load_device=None, disable_fast_fp8=False, fp8_optimizations=False, model_config=None):- fp8_compute = comfy.model_management.supports_fp8_compute(load_device) # TODO: if we support more ops this needs to be more granular- nvfp4_compute = comfy.model_management.supports_nvfp4_compute(load_device)- mxfp8_compute = comfy.model_management.supports_mxfp8_compute(load_device)- int8_compute = comfy.model_management.supports_int8_compute(load_device)+def get_disabled_quant_formats(device=None):+ disabled = set()+ if not comfy.model_management.supports_nvfp4_compute(device):+ disabled.add("nvfp4")+ if not comfy.model_management.supports_mxfp8_compute(device):+ disabled.add("mxfp8")+ if not comfy.model_management.supports_fp8_compute(device):+ disabled.add("float8_e4m3fn")+ disabled.add("float8_e5m2")+ if not comfy.model_management.supports_int8_compute(device):+ disabled.add("int8_tensorwise")+ disabled.add("convrot_w4a4")+ disabled.add("asym_w4a8_int8")+ return disabled+++def use_quantized_matmul(model, device):+ disabled = get_disabled_quant_formats(device)+ previous = []+ try:+ for module in model.modules():+ if isinstance(module, MixedPrecisionOp) and module.can_use_quantized_matmul(disabled):+ previous.append((module, module._full_precision_mm))+ module._full_precision_mm = False+ yield+ finally:+ for module, full_precision_mm in previous:+ module._full_precision_mm = full_precision_mm++def pick_operations(weight_dtype, compute_dtype, load_device=None, disable_fast_fp8=False, fp8_optimizations=False, model_config=None):if model_config and hasattr(model_config, 'quant_config') and model_config.quant_config:logging.info("Using mixed precision operations")- disabled = set()- if not nvfp4_compute:- disabled.add("nvfp4")- if not mxfp8_compute:- disabled.add("mxfp8")- if not fp8_compute:- disabled.add("float8_e4m3fn")- disabled.add("float8_e5m2")- if not int8_compute:- disabled.add("int8_tensorwise")- disabled.add("convrot_w4a4")- disabled.add("asym_w4a8_int8")+ disabled = get_disabled_quant_formats(load_device)logging.info("Native ops: {} {}".format(", ".join(QUANT_ALGOS.keys() - disabled), ", emulated ops: {}".format(", ".join(disabled)) if len(disabled) > 0 else ""))return mixed_precision_ops(model_config.quant_config, compute_dtype, disabled=disabled)+ fp8_compute = comfy.model_management.supports_fp8_compute(load_device)if (fp8_compute and(fp8_optimizations or PerformanceFeature.Fp8MatrixMultiplication in args.fast) anddiff --git a/comfy/sd.py b/comfy/sd.pyindex 54fbbd89c..1d736eebe 100644--- a/comfy/sd.py+++ b/comfy/sd.py@@ -475,7 +475,7 @@ class CLIP:self.cond_stage_model.set_clip_options({"layer": None})self.cond_stage_model.set_clip_options({"execution_device": device})- with model_management.cuda_device_context(device):+ with model_management.cuda_device_context(device), comfy.ops.use_quantized_matmul(self.cond_stage_model, device):return self.cond_stage_model.generate(tokens, do_sample=do_sample, max_length=max_length, temperature=temperature, top_k=top_k, top_p=top_p, min_p=min_p, repetition_penalty=repetition_penalty, seed=seed, presence_penalty=presence_penalty)def decode(self, token_ids, skip_special_tokens=True):diff --git a/comfy/text_encoders/ace15.py b/comfy/text_encoders/ace15.pyindex 3ad519314..ec8fa313e 100644--- a/comfy/text_encoders/ace15.py+++ b/comfy/text_encoders/ace15.py@@ -162,7 +162,8 @@ def generate_audio_codes(model, positive, negative, min_tokens=1, max_tokens=102else:ids = [positive]- return sample_manual_loop_no_classes(model, ids, cfg_scale=cfg_scale, temperature=temperature, top_p=top_p, top_k=top_k, min_p=min_p, seed=seed, min_tokens=min_tokens, max_new_tokens=max_tokens)+ with comfy.ops.use_quantized_matmul(model, model.execution_device):+ return sample_manual_loop_no_classes(model, ids, cfg_scale=cfg_scale, temperature=temperature, top_p=top_p, top_k=top_k, min_p=min_p, seed=seed, min_tokens=min_tokens, max_new_tokens=max_tokens)class ACE15Tokenizer(sd1_clip.SD1Tokenizer):