The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Support convrot int4 models. (#14859)
comfyanonymous73e84d5ec8b9
diff --git a/comfy/ops.py b/comfy/ops.pyindex 35a1ee31e..0c6fe4cb4 100644--- a/comfy/ops.py+++ b/comfy/ops.py@@ -1104,6 +1104,21 @@ def _load_quantized_module(module, super_load, state_dict, prefix, local_metadatscales["convrot_groupsize"] = int(layer_conf.get("convrot_groupsize", params_conf.get("convrot_groupsize", 256)))+ elif module.quant_format == "convrot_w4a4":+ scale = pop_scale("weight_scale")+ if scale is None:+ raise ValueError(f"Missing ConvRot W4A4 weight scale for layer {layer_name}")+ params_conf = layer_conf.get("params", {})+ if not isinstance(params_conf, dict):+ params_conf = {}+ scales = {+ "scale": scale,+ "convrot_groupsize": int(+ layer_conf.get("convrot_groupsize", params_conf.get("convrot_groupsize", 256))+ ),+ "quant_group_size": 64,+ "linear_dtype": layer_conf.get("linear_dtype", params_conf.get("linear_dtype", "int4")),+ }else:raise ValueError(f"Unsupported quantization format: {module.quant_format}")@@ -1150,6 +1165,11 @@ def _quantized_weight_state_dict(module, sd, prefix, extra_quant_conf=None, extrif module.quant_format == "int8_tensorwise" and getattr(params, "convrot", False):quant_conf["convrot"] = Truequant_conf["convrot_groupsize"] = getattr(params, "convrot_groupsize", 256)+ elif module.quant_format == "convrot_w4a4":+ quant_conf["convrot_groupsize"] = getattr(params, "convrot_groupsize", 256)+ linear_dtype = getattr(params, "linear_dtype", "int4")+ if linear_dtype != "int4":+ quant_conf["linear_dtype"] = linear_dtypeif extra_quant_conf:quant_conf.update(extra_quant_conf)sd[f"{prefix}comfy_quant"] = torch.tensor(list(json.dumps(quant_conf).encode("utf-8")), dtype=torch.uint8)@@ -1430,6 +1450,12 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_prec}if hasattr(params, "block_scale"): # NVFP4kwargs["block_scale"] = params.block_scale[i]+ if hasattr(params, "quant_group_size"):+ kwargs["quant_group_size"] = params.quant_group_size+ if hasattr(params, "convrot_groupsize"):+ kwargs["convrot_groupsize"] = params.convrot_groupsize+ if hasattr(params, "linear_dtype"):+ kwargs["linear_dtype"] = params.linear_dtypereturn QuantizedTensor(weight._qdata[i], weight._layout_cls, type(params)(**kwargs))def state_dict(self, *args, destination=None, prefix="", **kwargs):diff --git a/comfy/quant_ops.py b/comfy/quant_ops.pyindex 44f25a97e..53a0cb603 100644--- a/comfy/quant_ops.py+++ b/comfy/quant_ops.py@@ -10,6 +10,7 @@ try:QuantizedLayout,TensorCoreFP8Layout as _CKFp8Layout,TensorCoreNVFP4Layout as _CKNvfp4Layout,+ TensorCoreConvRotW4A4Layout as _CKTensorCoreConvRotW4A4Layout,TensorWiseINT8Layout as _CKTensorWiseINT8Layout,register_layout_op,register_layout_class,@@ -51,6 +52,9 @@ except ImportError as e:class _CKTensorWiseINT8Layout:pass+ class _CKTensorCoreConvRotW4A4Layout:+ pass+def register_layout_class(name, cls):pass@@ -179,6 +183,7 @@ class TensorCoreFP8E5M2Layout(_TensorCoreFP8LayoutBase):# Backward compatibility alias - default to E4M3TensorCoreFP8Layout = TensorCoreFP8E4M3LayoutTensorWiseINT8Layout = _CKTensorWiseINT8Layout+TensorCoreConvRotW4A4Layout = _CKTensorCoreConvRotW4A4Layout# ==============================================================================@@ -190,6 +195,7 @@ register_layout_class("TensorCoreFP8E4M3Layout", TensorCoreFP8E4M3Layout)register_layout_class("TensorCoreFP8E5M2Layout", TensorCoreFP8E5M2Layout)register_layout_class("TensorCoreNVFP4Layout", TensorCoreNVFP4Layout)register_layout_class("TensorWiseINT8Layout", _CKTensorWiseINT8Layout)+register_layout_class("TensorCoreConvRotW4A4Layout", _CKTensorCoreConvRotW4A4Layout)if _CK_MXFP8_AVAILABLE:register_layout_class("TensorCoreMXFP8Layout", TensorCoreMXFP8Layout)@@ -227,6 +233,13 @@ QUANT_ALGOS["int8_tensorwise"] = {"quantize_input": False,}+QUANT_ALGOS["convrot_w4a4"] = {+ "storage_t": torch.int8,+ "parameters": {"weight_scale"},+ "comfy_tensor_layout": "TensorCoreConvRotW4A4Layout",+ "quantize_input": False,+}+# ==============================================================================# Re-exports for backward compatibility@@ -239,6 +252,7 @@ __all__ = ["TensorCoreFP8E4M3Layout","TensorCoreFP8E5M2Layout","TensorCoreNVFP4Layout",+ "TensorCoreConvRotW4A4Layout","TensorWiseINT8Layout","QUANT_ALGOS","register_layout_op",diff --git a/requirements.txt b/requirements.txtindex e72f3045b..a8ea0eace 100644--- a/requirements.txt+++ b/requirements.txt@@ -22,7 +22,7 @@ alembicSQLAlchemy>=2.0.0filelockav>=16.0.0-comfy-kitchen==0.2.16+comfy-kitchen==0.2.17comfy-aimdo==0.4.10requestssimpleeval>=1.0.0diff --git a/tests-unit/comfy_quant/test_mixed_precision.py b/tests-unit/comfy_quant/test_mixed_precision.pyindex 43b4b7ce9..7bbc96616 100644--- a/tests-unit/comfy_quant/test_mixed_precision.py+++ b/tests-unit/comfy_quant/test_mixed_precision.py@@ -15,7 +15,7 @@ if not has_gpu():args.cpu = Truefrom comfy import ops-from comfy.quant_ops import QuantizedTensor+from comfy.quant_ops import QUANT_ALGOS, QuantizedTensorimport comfy.utils@@ -283,7 +283,59 @@ class TestMixedPrecisionOps(unittest.TestCase):saved = model.state_dict()saved_conf = json.loads(saved["layer.comfy_quant"].numpy().tobytes())self.assertTrue(saved_conf["convrot"])++ def test_convrot_w4a4_loads_into_params(self):+ """ConvRot W4A4 checkpoints must load as the dedicated kitchen layout."""+ if "convrot_w4a4" not in QUANT_ALGOS:+ self.skipTest("comfy_kitchen does not provide ConvRot W4A4")++ torch.manual_seed(456)+ layer_quant_config = {+ "layer": {+ "format": "convrot_w4a4",+ "convrot_groupsize": 256,+ "linear_dtype": "int8",+ }+ }+ weight = torch.randn(16, 256, dtype=torch.bfloat16)+ bias = torch.randn(16, dtype=torch.bfloat16)+ q_weight = QuantizedTensor.from_float(+ weight,+ "TensorCoreConvRotW4A4Layout",+ convrot_groupsize=256,+ quant_group_size=64,+ )+ state_dict = {+ "layer.weight": q_weight._qdata,+ "layer.bias": bias,+ "layer.weight_scale": q_weight._params.scale,+ }++ state_dict, _ = comfy.utils.convert_old_quants(+ state_dict,+ metadata={"_quantization_metadata": json.dumps({"layers": layer_quant_config})},+ )+ model = torch.nn.Module()+ model.layer = ops.mixed_precision_ops({}).Linear(256, 16, device="cpu", dtype=torch.bfloat16)+ model.load_state_dict(state_dict, strict=False)++ self.assertIsInstance(model.layer.weight, QuantizedTensor)+ self.assertEqual(model.layer.weight._layout_cls, "TensorCoreConvRotW4A4Layout")+ self.assertEqual(model.layer.weight._params.convrot_groupsize, 256)+ self.assertEqual(model.layer.weight._params.quant_group_size, 64)+ self.assertEqual(model.layer.weight._params.linear_dtype, "int8")++ input_tensor = torch.randn(4, 256, dtype=torch.bfloat16)+ loaded_out = model.layer(input_tensor)+ ref_out = torch.nn.functional.linear(input_tensor, q_weight, bias)+ self.assertTrue(torch.equal(loaded_out, ref_out))++ saved = model.state_dict()+ saved_conf = json.loads(saved["layer.comfy_quant"].numpy().tobytes())+ self.assertEqual(saved_conf["format"], "convrot_w4a4")self.assertEqual(saved_conf["convrot_groupsize"], 256)+ self.assertEqual(saved_conf["linear_dtype"], "int8")+ self.assertNotIn("quant_group_size", saved_conf)if __name__ == "__main__":unittest.main()