Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Disable int8 weight-only quantization on devices without torch._int_mm (fixes MPS crash) (#16130)

Jialong(Bruce) Li313a76fb8d83
diff --git a/comfy/controlnet.py b/comfy/controlnet.py
index 7e35fe027..8072304aa 100644
--- a/comfy/controlnet.py
+++ b/comfy/controlnet.py
@@ -501,7 +501,7 @@ def controlnet_config(sd, model_options={}):
operations = model_options.get("custom_operations", None)
if operations is None:
- operations = comfy.ops.pick_operations(unet_dtype, manual_cast_dtype, disable_fast_fp8=True)
+ operations = comfy.ops.pick_operations(unet_dtype, manual_cast_dtype, load_device=load_device, disable_fast_fp8=True)
offload_device = comfy.model_management.unet_offload_device()
return model_config, operations, load_device, unet_dtype, manual_cast_dtype, offload_device
@@ -585,7 +585,7 @@ def load_controlnet_sd35(sd, model_options={}):
operations = model_options.get("custom_operations", None)
if operations is None:
- operations = comfy.ops.pick_operations(unet_dtype, manual_cast_dtype, disable_fast_fp8=True)
+ operations = comfy.ops.pick_operations(unet_dtype, manual_cast_dtype, load_device=load_device, disable_fast_fp8=True)
control_model = comfy.cldm.dit_embedder.ControlNetEmbedder(img_size=None,
patch_size=2,
@@ -683,7 +683,7 @@ def load_controlnet_qwen_fun(sd, model_options={}):
operations = model_options.get("custom_operations", None)
if operations is None:
- operations = comfy.ops.pick_operations(unet_dtype, manual_cast_dtype, disable_fast_fp8=True)
+ operations = comfy.ops.pick_operations(unet_dtype, manual_cast_dtype, load_device=load_device, disable_fast_fp8=True)
in_features = sd["control_img_in.weight"].shape[1]
inner_dim = sd["control_img_in.weight"].shape[0]
@@ -838,7 +838,7 @@ def load_controlnet_state_dict(state_dict, model=None, model_options={}):
manual_cast_dtype = comfy.model_management.unet_manual_cast(unet_dtype, load_device)
operations = model_options.get("custom_operations", None)
if operations is None:
- operations = comfy.ops.pick_operations(unet_dtype, manual_cast_dtype)
+ operations = comfy.ops.pick_operations(unet_dtype, manual_cast_dtype, load_device=load_device, disable_fast_fp8=True)
controlnet_config["operations"] = operations
controlnet_config["dtype"] = unet_dtype
diff --git a/comfy/model_management.py b/comfy/model_management.py
index 1eead636e..a67626cc9 100644
--- a/comfy/model_management.py
+++ b/comfy/model_management.py
@@ -2022,6 +2022,24 @@ def supports_fp64(device=None):
return True
+def supports_int8_compute(device=None):
+ # The eager comfy_kitchen backend implements int8 weight-only quantized
+ # matmul via torch._int_mm, which PyTorch does not implement for MPS.
+ # https://github.com/pytorch/pytorch/issues/141287
+ if (device is not None and is_device_mps(device)) or mps_mode():
+ return False
+
+ if is_intel_xpu():
+ return False
+
+ if is_directml_enabled():
+ return False
+
+ if is_ixuca():
+ return False
+
+ return True
+
def extended_fp16_support():
# TODO: check why some models work with fp16 on newer torch versions but not on older
if torch_version_numeric < (2, 7):
diff --git a/comfy/ops.py b/comfy/ops.py
index ff64aad59..d9df909bb 100644
--- a/comfy/ops.py
+++ b/comfy/ops.py
@@ -1342,6 +1342,12 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec
compute_dtype=compute_dtype,
want_requant=want_requant,
) as (weight, bias):
+ if self._full_precision_mm and isinstance(weight, QuantizedTensor):
+ # cast_bias_weight only dequantizes on a dtype change, which is a
+ # no-op here when the quantized weight's orig_dtype already equals
+ # the compute dtype. Force it so the disabled/unsupported-format
+ # fallback doesn't hand a QuantizedTensor to a plain linear() call.
+ weight = weight.dequantize()
return self._forward(input, weight, bias)
with CastBiasWeightContext(
@@ -1652,6 +1658,7 @@ def pick_operations(weight_dtype, compute_dtype, load_device=None, disable_fast_
fp8_compute = comfy.model_management.supports_fp8_compute(load_device) # TODO: if we support more ops this needs to be more granular
nvfp4_compute = comfy.model_management.supports_nvfp4_compute(load_device)
mxfp8_compute = comfy.model_management.supports_mxfp8_compute(load_device)
+ int8_compute = comfy.model_management.supports_int8_compute(load_device)
if model_config and hasattr(model_config, 'quant_config') and model_config.quant_config:
logging.info("Using mixed precision operations")
@@ -1663,6 +1670,10 @@ def pick_operations(weight_dtype, compute_dtype, load_device=None, disable_fast_
if not fp8_compute:
disabled.add("float8_e4m3fn")
disabled.add("float8_e5m2")
+ if not int8_compute:
+ disabled.add("int8_tensorwise")
+ disabled.add("convrot_w4a4")
+ disabled.add("asym_w4a8_int8")
logging.info("Native ops: {} {}".format(", ".join(QUANT_ALGOS.keys() - disabled), ", emulated ops: {}".format(", ".join(disabled)) if len(disabled) > 0 else ""))
return mixed_precision_ops(model_config.quant_config, compute_dtype, disabled=disabled)
diff --git a/comfy_extras/nodes_model_patch.py b/comfy_extras/nodes_model_patch.py
index 82a5f58eb..eb461de63 100644
--- a/comfy_extras/nodes_model_patch.py
+++ b/comfy_extras/nodes_model_patch.py
@@ -283,7 +283,7 @@ class ModelPatchLoader:
)
manual_cast_dtype = comfy.model_management.unet_manual_cast(
dtype, load_device, supported_dtypes=[torch.bfloat16, torch.float32])
- operations = comfy.ops.pick_operations(dtype, manual_cast_dtype)
+ operations = comfy.ops.pick_operations(dtype, manual_cast_dtype, load_device=load_device)
num_blocks = 0
while "control_blocks.{}.after_proj.weight".format(num_blocks) in sd:
diff --git a/tests-unit/comfy_quant/test_mixed_precision.py b/tests-unit/comfy_quant/test_mixed_precision.py
index 7bbc96616..30a446539 100644
--- a/tests-unit/comfy_quant/test_mixed_precision.py
+++ b/tests-unit/comfy_quant/test_mixed_precision.py
@@ -3,6 +3,7 @@ import torch
import sys
import os
import json
+from types import SimpleNamespace
# Add comfy to path
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))
@@ -284,6 +285,75 @@ class TestMixedPrecisionOps(unittest.TestCase):
saved_conf = json.loads(saved["layer.comfy_quant"].numpy().tobytes())
self.assertTrue(saved_conf["convrot"])
+ def test_int8_disabled_on_unsupported_device_falls_back_to_full_precision(self):
+ """On a device that can't run comfy_kitchen's fast int8 matmul (e.g. MPS,
+ which lacks aten::_int_mm), pick_operations must mark int8 formats as
+ disabled so layers dequantize instead of taking the fast quantized path."""
+ import comfy.model_management as mm
+
+ orig_supports_int8 = mm.supports_int8_compute
+ mm.supports_int8_compute = lambda device=None: False
+ try:
+ model_config = SimpleNamespace(quant_config={"layer": {"format": "int8_tensorwise"}})
+ operations = ops.pick_operations(torch.bfloat16, torch.bfloat16, model_config=model_config)
+
+ torch.manual_seed(789)
+ weight = torch.randn(16, 256, dtype=torch.bfloat16)
+ bias = torch.randn(16, dtype=torch.bfloat16)
+ q_weight = QuantizedTensor.from_float(weight, "TensorWiseINT8Layout", per_channel=True)
+ state_dict = {
+ "layer.weight": q_weight._qdata,
+ "layer.bias": bias,
+ "layer.weight_scale": q_weight._params.scale,
+ }
+ layer_quant_config = {"layer": {"format": "int8_tensorwise"}}
+ state_dict, _ = comfy.utils.convert_old_quants(
+ state_dict,
+ metadata={"_quantization_metadata": json.dumps({"layers": layer_quant_config})},
+ )
+
+ model = torch.nn.Module()
+ model.layer = operations.Linear(256, 16, device="cpu", dtype=torch.bfloat16)
+ model.load_state_dict(state_dict, strict=False)
+
+ self.assertIsInstance(model.layer.weight, QuantizedTensor)
+ # The layer must be forced onto the full-precision (dequantized)
+ # path since the fast int8 path isn't usable on this device.
+ self.assertTrue(model.layer._full_precision_mm)
+
+ # The weight's orig_dtype matches the compute dtype here (both bfloat16),
+ # so cast_bias_weight's dtype-change check alone won't dequantize it. Confirm
+ # the module still hands a real Tensor (not a QuantizedTensor) to the plain
+ # linear() call, since dispatching a QuantizedTensor there would route back
+ # into the disabled fast int8 matmul instead of the full-precision fallback.
+ seen_weight_types = []
+ orig_module_forward = model.layer._forward
+ def _capturing_forward(input, weight, bias, _orig=orig_module_forward):
+ seen_weight_types.append(type(weight))
+ return _orig(input, weight, bias)
+ model.layer._forward = _capturing_forward
+
+ input_tensor = torch.randn(4, 256, dtype=torch.bfloat16)
+ output = model.layer(input_tensor)
+ self.assertEqual(output.shape, (4, 16))
+ self.assertEqual(seen_weight_types, [torch.Tensor])
+ finally:
+ mm.supports_int8_compute = orig_supports_int8
+
+ def test_supports_int8_compute_treats_mps_mode_as_unsupported_when_device_is_none(self):
+ """Call sites (like pick_operations' default) may omit load_device. On an
+ MPS machine that must still report int8 as unsupported instead of
+ silently defaulting to True, matching supports_fp64's handling of the
+ same device=None case (see Comfy-Org/ComfyUI#16136)."""
+ import comfy.model_management as mm
+
+ orig_cpu_state = mm.cpu_state
+ mm.cpu_state = mm.CPUState.MPS
+ try:
+ self.assertFalse(mm.supports_int8_compute(None))
+ finally:
+ mm.cpu_state = orig_cpu_state
+
def test_convrot_w4a4_loads_into_params(self):
"""ConvRot W4A4 checkpoints must load as the dedicated kitchen layout."""
if "convrot_w4a4" not in QUANT_ALGOS: