The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Disable int8 weight-only quantization on devices without torch._int_mm (fixes MPS crash) (#16130)
Jialong(Bruce) Li313a76fb8d83
diff --git a/comfy/controlnet.py b/comfy/controlnet.pyindex 7e35fe027..8072304aa 100644--- a/comfy/controlnet.py+++ b/comfy/controlnet.py@@ -501,7 +501,7 @@ def controlnet_config(sd, model_options={}):operations = model_options.get("custom_operations", None)if operations is None:- operations = comfy.ops.pick_operations(unet_dtype, manual_cast_dtype, disable_fast_fp8=True)+ operations = comfy.ops.pick_operations(unet_dtype, manual_cast_dtype, load_device=load_device, disable_fast_fp8=True)offload_device = comfy.model_management.unet_offload_device()return model_config, operations, load_device, unet_dtype, manual_cast_dtype, offload_device@@ -585,7 +585,7 @@ def load_controlnet_sd35(sd, model_options={}):operations = model_options.get("custom_operations", None)if operations is None:- operations = comfy.ops.pick_operations(unet_dtype, manual_cast_dtype, disable_fast_fp8=True)+ operations = comfy.ops.pick_operations(unet_dtype, manual_cast_dtype, load_device=load_device, disable_fast_fp8=True)control_model = comfy.cldm.dit_embedder.ControlNetEmbedder(img_size=None,patch_size=2,@@ -683,7 +683,7 @@ def load_controlnet_qwen_fun(sd, model_options={}):operations = model_options.get("custom_operations", None)if operations is None:- operations = comfy.ops.pick_operations(unet_dtype, manual_cast_dtype, disable_fast_fp8=True)+ operations = comfy.ops.pick_operations(unet_dtype, manual_cast_dtype, load_device=load_device, disable_fast_fp8=True)in_features = sd["control_img_in.weight"].shape[1]inner_dim = sd["control_img_in.weight"].shape[0]@@ -838,7 +838,7 @@ def load_controlnet_state_dict(state_dict, model=None, model_options={}):manual_cast_dtype = comfy.model_management.unet_manual_cast(unet_dtype, load_device)operations = model_options.get("custom_operations", None)if operations is None:- operations = comfy.ops.pick_operations(unet_dtype, manual_cast_dtype)+ operations = comfy.ops.pick_operations(unet_dtype, manual_cast_dtype, load_device=load_device, disable_fast_fp8=True)controlnet_config["operations"] = operationscontrolnet_config["dtype"] = unet_dtypediff --git a/comfy/model_management.py b/comfy/model_management.pyindex 1eead636e..a67626cc9 100644--- a/comfy/model_management.py+++ b/comfy/model_management.py@@ -2022,6 +2022,24 @@ def supports_fp64(device=None):return True+def supports_int8_compute(device=None):+ # The eager comfy_kitchen backend implements int8 weight-only quantized+ # matmul via torch._int_mm, which PyTorch does not implement for MPS.+ # https://github.com/pytorch/pytorch/issues/141287+ if (device is not None and is_device_mps(device)) or mps_mode():+ return False++ if is_intel_xpu():+ return False++ if is_directml_enabled():+ return False++ if is_ixuca():+ return False++ return True+def extended_fp16_support():# TODO: check why some models work with fp16 on newer torch versions but not on olderif torch_version_numeric < (2, 7):diff --git a/comfy/ops.py b/comfy/ops.pyindex ff64aad59..d9df909bb 100644--- a/comfy/ops.py+++ b/comfy/ops.py@@ -1342,6 +1342,12 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_preccompute_dtype=compute_dtype,want_requant=want_requant,) as (weight, bias):+ if self._full_precision_mm and isinstance(weight, QuantizedTensor):+ # cast_bias_weight only dequantizes on a dtype change, which is a+ # no-op here when the quantized weight's orig_dtype already equals+ # the compute dtype. Force it so the disabled/unsupported-format+ # fallback doesn't hand a QuantizedTensor to a plain linear() call.+ weight = weight.dequantize()return self._forward(input, weight, bias)with CastBiasWeightContext(@@ -1652,6 +1658,7 @@ def pick_operations(weight_dtype, compute_dtype, load_device=None, disable_fast_fp8_compute = comfy.model_management.supports_fp8_compute(load_device) # TODO: if we support more ops this needs to be more granularnvfp4_compute = comfy.model_management.supports_nvfp4_compute(load_device)mxfp8_compute = comfy.model_management.supports_mxfp8_compute(load_device)+ int8_compute = comfy.model_management.supports_int8_compute(load_device)if model_config and hasattr(model_config, 'quant_config') and model_config.quant_config:logging.info("Using mixed precision operations")@@ -1663,6 +1670,10 @@ def pick_operations(weight_dtype, compute_dtype, load_device=None, disable_fast_if not fp8_compute:disabled.add("float8_e4m3fn")disabled.add("float8_e5m2")+ if not int8_compute:+ disabled.add("int8_tensorwise")+ disabled.add("convrot_w4a4")+ disabled.add("asym_w4a8_int8")logging.info("Native ops: {} {}".format(", ".join(QUANT_ALGOS.keys() - disabled), ", emulated ops: {}".format(", ".join(disabled)) if len(disabled) > 0 else ""))return mixed_precision_ops(model_config.quant_config, compute_dtype, disabled=disabled)diff --git a/comfy_extras/nodes_model_patch.py b/comfy_extras/nodes_model_patch.pyindex 82a5f58eb..eb461de63 100644--- a/comfy_extras/nodes_model_patch.py+++ b/comfy_extras/nodes_model_patch.py@@ -283,7 +283,7 @@ class ModelPatchLoader:)manual_cast_dtype = comfy.model_management.unet_manual_cast(dtype, load_device, supported_dtypes=[torch.bfloat16, torch.float32])- operations = comfy.ops.pick_operations(dtype, manual_cast_dtype)+ operations = comfy.ops.pick_operations(dtype, manual_cast_dtype, load_device=load_device)num_blocks = 0while "control_blocks.{}.after_proj.weight".format(num_blocks) in sd:diff --git a/tests-unit/comfy_quant/test_mixed_precision.py b/tests-unit/comfy_quant/test_mixed_precision.pyindex 7bbc96616..30a446539 100644--- a/tests-unit/comfy_quant/test_mixed_precision.py+++ b/tests-unit/comfy_quant/test_mixed_precision.py@@ -3,6 +3,7 @@ import torchimport sysimport osimport json+from types import SimpleNamespace# Add comfy to pathsys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))@@ -284,6 +285,75 @@ class TestMixedPrecisionOps(unittest.TestCase):saved_conf = json.loads(saved["layer.comfy_quant"].numpy().tobytes())self.assertTrue(saved_conf["convrot"])+ def test_int8_disabled_on_unsupported_device_falls_back_to_full_precision(self):+ """On a device that can't run comfy_kitchen's fast int8 matmul (e.g. MPS,+ which lacks aten::_int_mm), pick_operations must mark int8 formats as+ disabled so layers dequantize instead of taking the fast quantized path."""+ import comfy.model_management as mm++ orig_supports_int8 = mm.supports_int8_compute+ mm.supports_int8_compute = lambda device=None: False+ try:+ model_config = SimpleNamespace(quant_config={"layer": {"format": "int8_tensorwise"}})+ operations = ops.pick_operations(torch.bfloat16, torch.bfloat16, model_config=model_config)++ torch.manual_seed(789)+ weight = torch.randn(16, 256, dtype=torch.bfloat16)+ bias = torch.randn(16, dtype=torch.bfloat16)+ q_weight = QuantizedTensor.from_float(weight, "TensorWiseINT8Layout", per_channel=True)+ state_dict = {+ "layer.weight": q_weight._qdata,+ "layer.bias": bias,+ "layer.weight_scale": q_weight._params.scale,+ }+ layer_quant_config = {"layer": {"format": "int8_tensorwise"}}+ state_dict, _ = comfy.utils.convert_old_quants(+ state_dict,+ metadata={"_quantization_metadata": json.dumps({"layers": layer_quant_config})},+ )++ model = torch.nn.Module()+ model.layer = operations.Linear(256, 16, device="cpu", dtype=torch.bfloat16)+ model.load_state_dict(state_dict, strict=False)++ self.assertIsInstance(model.layer.weight, QuantizedTensor)+ # The layer must be forced onto the full-precision (dequantized)+ # path since the fast int8 path isn't usable on this device.+ self.assertTrue(model.layer._full_precision_mm)++ # The weight's orig_dtype matches the compute dtype here (both bfloat16),+ # so cast_bias_weight's dtype-change check alone won't dequantize it. Confirm+ # the module still hands a real Tensor (not a QuantizedTensor) to the plain+ # linear() call, since dispatching a QuantizedTensor there would route back+ # into the disabled fast int8 matmul instead of the full-precision fallback.+ seen_weight_types = []+ orig_module_forward = model.layer._forward+ def _capturing_forward(input, weight, bias, _orig=orig_module_forward):+ seen_weight_types.append(type(weight))+ return _orig(input, weight, bias)+ model.layer._forward = _capturing_forward++ input_tensor = torch.randn(4, 256, dtype=torch.bfloat16)+ output = model.layer(input_tensor)+ self.assertEqual(output.shape, (4, 16))+ self.assertEqual(seen_weight_types, [torch.Tensor])+ finally:+ mm.supports_int8_compute = orig_supports_int8++ def test_supports_int8_compute_treats_mps_mode_as_unsupported_when_device_is_none(self):+ """Call sites (like pick_operations' default) may omit load_device. On an+ MPS machine that must still report int8 as unsupported instead of+ silently defaulting to True, matching supports_fp64's handling of the+ same device=None case (see Comfy-Org/ComfyUI#16136)."""+ import comfy.model_management as mm++ orig_cpu_state = mm.cpu_state+ mm.cpu_state = mm.CPUState.MPS+ try:+ self.assertFalse(mm.supports_int8_compute(None))+ finally:+ mm.cpu_state = orig_cpu_state+def test_convrot_w4a4_loads_into_params(self):"""ConvRot W4A4 checkpoints must load as the dedicated kitchen layout."""if "convrot_w4a4" not in QUANT_ALGOS: