Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Cleaner way of enabling quantized mm on text gen but not on text enc. (#16189)

comfyanonymous249c5a3b9518
diff --git a/comfy/ops.py b/comfy/ops.py
index d53886de0..24631840a 100644
--- a/comfy/ops.py
+++ b/comfy/ops.py
@@ -1287,6 +1287,16 @@ def _quantized_weight_state_dict(module, sd, prefix, extra_quant_conf=None, extr
return sd
+class MixedPrecisionOp(CastWeightBiasOp):
+ quant_format = None
+
+ def can_use_quantized_matmul(self, disabled_formats):
+ return (self.quant_format in QUANT_ALGOS
+ and not self._full_precision_mm_config
+ and self.quant_format not in self._disabled_formats
+ and self.quant_format not in disabled_formats)
+
+
def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_precision_mm=False, disabled=[]):
class MixedPrecisionOps(manual_cast):
_quant_config = quant_config
@@ -1294,7 +1304,7 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec
_full_precision_mm = full_precision_mm
_disabled = disabled
- class Linear(torch.nn.Module, CastWeightBiasOp):
+ class Linear(torch.nn.Module, MixedPrecisionOp):
_disabled_formats = disabled
def __init__(self, in_features: int, out_features: int, bias: bool = True, device=None, dtype=None):
@@ -1342,8 +1352,7 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec
compute_dtype=compute_dtype,
want_requant=want_requant,
) as (weight, bias):
- if isinstance(weight, QuantizedTensor) and (self._full_precision_mm_config or getattr(self, "quant_format", None) in self._disabled_formats or not weight.layout_cls.supports_fast_matmul()):
- # explicit per-layer full precision, or a format this device can't run: don't reach the fast quantized matmul
+ if self._full_precision_mm and isinstance(weight, QuantizedTensor):
weight = weight.dequantize()
return self._forward(input, weight, bias)
@@ -1447,7 +1456,7 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec
def _apply(self, fn, recurse=True): # This is to get torch.compile + moving weights to another device working
return _quantized_apply(self, fn, recurse)
- class MoEExperts(torch.nn.Module, CastWeightBiasOp):
+ class MoEExperts(torch.nn.Module, MixedPrecisionOp):
"""Container for E quantized expert weights, indexed via expert_weight(i).
The bank lives on self.weight as a single 3D tensor — either a
@@ -1651,29 +1660,45 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec
return MixedPrecisionOps
-def pick_operations(weight_dtype, compute_dtype, load_device=None, disable_fast_fp8=False, fp8_optimizations=False, model_config=None):
- fp8_compute = comfy.model_management.supports_fp8_compute(load_device) # TODO: if we support more ops this needs to be more granular
- nvfp4_compute = comfy.model_management.supports_nvfp4_compute(load_device)
- mxfp8_compute = comfy.model_management.supports_mxfp8_compute(load_device)
- int8_compute = comfy.model_management.supports_int8_compute(load_device)
+def get_disabled_quant_formats(device=None):
+ disabled = set()
+ if not comfy.model_management.supports_nvfp4_compute(device):
+ disabled.add("nvfp4")
+ if not comfy.model_management.supports_mxfp8_compute(device):
+ disabled.add("mxfp8")
+ if not comfy.model_management.supports_fp8_compute(device):
+ disabled.add("float8_e4m3fn")
+ disabled.add("float8_e5m2")
+ if not comfy.model_management.supports_int8_compute(device):
+ disabled.add("int8_tensorwise")
+ disabled.add("convrot_w4a4")
+ disabled.add("asym_w4a8_int8")
+ return disabled
+
+
+def use_quantized_matmul(model, device):
+ disabled = get_disabled_quant_formats(device)
+ previous = []
+ try:
+ for module in model.modules():
+ if isinstance(module, MixedPrecisionOp) and module.can_use_quantized_matmul(disabled):
+ previous.append((module, module._full_precision_mm))
+ module._full_precision_mm = False
+ yield
+ finally:
+ for module, full_precision_mm in previous:
+ module._full_precision_mm = full_precision_mm
+
+def pick_operations(weight_dtype, compute_dtype, load_device=None, disable_fast_fp8=False, fp8_optimizations=False, model_config=None):
if model_config and hasattr(model_config, 'quant_config') and model_config.quant_config:
logging.info("Using mixed precision operations")
- disabled = set()
- if not nvfp4_compute:
- disabled.add("nvfp4")
- if not mxfp8_compute:
- disabled.add("mxfp8")
- if not fp8_compute:
- disabled.add("float8_e4m3fn")
- disabled.add("float8_e5m2")
- if not int8_compute:
- disabled.add("int8_tensorwise")
- disabled.add("convrot_w4a4")
- disabled.add("asym_w4a8_int8")
+ disabled = get_disabled_quant_formats(load_device)
logging.info("Native ops: {} {}".format(", ".join(QUANT_ALGOS.keys() - disabled), ", emulated ops: {}".format(", ".join(disabled)) if len(disabled) > 0 else ""))
return mixed_precision_ops(model_config.quant_config, compute_dtype, disabled=disabled)
+ fp8_compute = comfy.model_management.supports_fp8_compute(load_device)
if (
fp8_compute and
(fp8_optimizations or PerformanceFeature.Fp8MatrixMultiplication in args.fast) and
diff --git a/comfy/sd.py b/comfy/sd.py
index 54fbbd89c..1d736eebe 100644
--- a/comfy/sd.py
+++ b/comfy/sd.py
@@ -475,7 +475,7 @@ class CLIP:
self.cond_stage_model.set_clip_options({"layer": None})
self.cond_stage_model.set_clip_options({"execution_device": device})
- with model_management.cuda_device_context(device):
+ with model_management.cuda_device_context(device), comfy.ops.use_quantized_matmul(self.cond_stage_model, device):
return self.cond_stage_model.generate(tokens, do_sample=do_sample, max_length=max_length, temperature=temperature, top_k=top_k, top_p=top_p, min_p=min_p, repetition_penalty=repetition_penalty, seed=seed, presence_penalty=presence_penalty)
def decode(self, token_ids, skip_special_tokens=True):
diff --git a/comfy/text_encoders/ace15.py b/comfy/text_encoders/ace15.py
index 3ad519314..ec8fa313e 100644
--- a/comfy/text_encoders/ace15.py
+++ b/comfy/text_encoders/ace15.py
@@ -162,7 +162,8 @@ def generate_audio_codes(model, positive, negative, min_tokens=1, max_tokens=102
else:
ids = [positive]
- return sample_manual_loop_no_classes(model, ids, cfg_scale=cfg_scale, temperature=temperature, top_p=top_p, top_k=top_k, min_p=min_p, seed=seed, min_tokens=min_tokens, max_new_tokens=max_tokens)
+ with comfy.ops.use_quantized_matmul(model, model.execution_device):
+ return sample_manual_loop_no_classes(model, ids, cfg_scale=cfg_scale, temperature=temperature, top_p=top_p, top_k=top_k, min_p=min_p, seed=seed, min_tokens=min_tokens, max_new_tokens=max_tokens)
class ACE15Tokenizer(sd1_clip.SD1Tokenizer):