The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Support w6a8 quantization format (#16483)
Jukka Seppänen491bf3b928db
diff --git a/comfy/ops.py b/comfy/ops.pyindex 860e09ea0..ff12dba89 100644--- a/comfy/ops.py+++ b/comfy/ops.py@@ -1039,6 +1039,11 @@ def linear_input_act(linear, x, input_act, act_weight=None, act_eps=0.0,uncast_bias_weight(linear, weight, bias, offload_stream)+# Grouped-scale integer weights that run through the shared INT8 GEMM (comfy_kitchen+# AsymW4A8Int8Layout): format -> code width, which also fixes the packed weight width K*bits/8.+_GROUPED_INT8_FORMATS = {"asym_w4a8_int8": 4, "w6a8_int8": 6}++class QuantLinearFunc(torch.autograd.Function):"""Custom autograd function for quantized linear: quantized forward, optionally FP8 backward.@@ -1249,12 +1254,19 @@ def _load_quantized_module(module, super_load, state_dict, prefix, local_metadat"quant_group_size": 64,"linear_dtype": layer_conf.get("linear_dtype", params_conf.get("linear_dtype", "int4")),}- elif module.quant_format == "asym_w4a8_int8":- # int4 weight (packed int8 [N,K/2]) + fp8 per-group scale (weight_s_rel),- # fp32 per-channel scale (weight_s_channel) + optional Lloyd-Max codebook.+ elif module.quant_format in _GROUPED_INT8_FORMATS:+ # asym_w4a8_int8: int4 weight (packed int8 [N,K/2]) + optional Lloyd-Max codebook;+ # w6a8_int8: uniform int6 weight (packed int8 [N,3K/4]). Both carry an fp8 per-group+ # scale (weight_s_rel) and an fp32 per-channel scale (weight_s_channel).+ bits = _GROUPED_INT8_FORMATS[module.quant_format]+ if weight.shape[1] * 8 != module._orig_shape[1] * bits:+ raise ValueError(+ f"{module.quant_format} layer {layer_name}: packed weight width {weight.shape[1]} "+ f"does not match K={module._orig_shape[1]} at {bits} bits"+ )scale = pop_scale("weight_s_rel")if scale is None:- raise ValueError(f"Missing W4A8 group scale (weight_s_rel) for layer {layer_name}")+ raise ValueError(f"Missing {module.quant_format} group scale (weight_s_rel) for layer {layer_name}")if scale.dtype == torch.uint8:scale = scale.view(torch.float8_e4m3fn)params_conf = layer_conf.get("params", {})@@ -1320,7 +1332,7 @@ def _quantized_weight_state_dict(module, sd, prefix, extra_quant_conf=None, extrlinear_dtype = getattr(params, "linear_dtype", "int4")if linear_dtype != "int4":quant_conf["linear_dtype"] = linear_dtype- elif module.quant_format == "asym_w4a8_int8":+ elif module.quant_format in _GROUPED_INT8_FORMATS:quant_conf["group_size"] = getattr(params, "group_size", 16)quant_conf["convrot_groupsize"] = getattr(params, "convrot_groupsize", 256)if extra_quant_conf:@@ -1759,7 +1771,7 @@ def get_disabled_quant_formats(device=None):if not comfy.model_management.supports_int8_compute(device):disabled.add("int8_tensorwise")disabled.add("convrot_w4a4")- disabled.add("asym_w4a8_int8")+ disabled.update(_GROUPED_INT8_FORMATS)return disableddiff --git a/comfy/quant_ops.py b/comfy/quant_ops.pyindex 01dfff370..0b0f4f863 100644--- a/comfy/quant_ops.py+++ b/comfy/quant_ops.py@@ -256,6 +256,14 @@ QUANT_ALGOS["asym_w4a8_int8"] = {"quantize_input": False,}+# Same layout class and kernels, 6-bit uniform codes: weight is int8 [N, 3K/4], no codebook.+QUANT_ALGOS["w6a8_int8"] = {+ "storage_t": torch.int8,+ "parameters": {"weight_scale"},+ "comfy_tensor_layout": "AsymW4A8Int8Layout",+ "quantize_input": False,+}+# ==============================================================================# Re-exports for backward compatibility