Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Enhance quantization configuration support for mixed precision and schemes in utils and tests (#1643)

Xin He4632b2e9f339
diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.py
index 03d4902bf..58f64f683 100644
--- a/auto_round/compressors/utils.py
+++ b/auto_round/compressors/utils.py
@@ -321,6 +321,12 @@ def set_layer_config(
elif isinstance(item, QuantizationScheme):
config = asdict(item)
elif isinstance(item, dict):
+ # Support "scheme" key inside dict: resolve the preset and merge overrides
+ if "scheme" in item:
+ scheme_name = item.pop("scheme")
+ base = asdict(preset_name_to_scheme(scheme_name.upper()))
+ base.update(item)
+ item = base
invalid = set(item) - set(scheme_keys + ("fixed_by_user", "scale_dtype"))
if invalid:
raise ValueError(
diff --git a/auto_round/export/export_to_llmcompressor/config.py b/auto_round/export/export_to_llmcompressor/config.py
index 6d6513d80..546b9addb 100644
--- a/auto_round/export/export_to_llmcompressor/config.py
+++ b/auto_round/export/export_to_llmcompressor/config.py
@@ -63,13 +63,16 @@ def initialize_quantization(scheme, targets=["Linear"], config_groups=None, kv_c
config_groups = config_groups
kv_cache_scheme = kv_cache_scheme
ignore = ignore
+ using_mxfp4_for_mxfp8 = False
check_compressed_tensors_supported()
if scheme is not None and config_groups is not None:
raise ValueError("Please specify either `scheme` or `config_groups`")
if scheme is not None:
# takes precedence over config_groups
-
+ if not is_preset_scheme(scheme) and scheme.lower() == "mxfp8":
+ scheme = "mxfp4"
+ using_mxfp4_for_mxfp8 = True # mxfp8 is not supported.
if isinstance(scheme, str) and is_preset_scheme(scheme):
# attach targets to scheme
scheme = {scheme: targets}
@@ -80,6 +83,10 @@ def initialize_quantization(scheme, targets=["Linear"], config_groups=None, kv_c
scheme = preset_name_to_scheme(key, scheme[key])
else:
scheme = QuantizationScheme.model_validate({"targets": scheme[key], **scheme})
+ if using_mxfp4_for_mxfp8:
+ scheme.weights.num_bits = 8
+ scheme.input_activations.num_bits = 8
+ scheme.format = "mxfp8-quantized"
group_name = f"group_{idx}"
config_groups[group_name] = scheme
diff --git a/auto_round/export/export_to_llmcompressor/export_to_fp.py b/auto_round/export/export_to_llmcompressor/export_to_fp.py
index 05c01ee3a..165903396 100644
--- a/auto_round/export/export_to_llmcompressor/export_to_fp.py
+++ b/auto_round/export/export_to_llmcompressor/export_to_fp.py
@@ -16,6 +16,7 @@ import copy
import inspect
import json
import os
+import sys
from concurrent.futures import ThreadPoolExecutor
from typing import Callable, Union
@@ -27,6 +28,7 @@ from tqdm import tqdm
from auto_round.compressors.utils import is_mx_fp, is_nv_fp
from auto_round.export.export_to_autoround.qlinear_fp import QuantLinear
+from auto_round.export.export_to_llmcompressor.config import initialize_quantization
from auto_round.export.export_to_llmcompressor.utils import generate_ignore_regex_list
from auto_round.export.utils import filter_quantization_config, release_layer_safely, save_model
from auto_round.logger import logger
@@ -36,6 +38,7 @@ from auto_round.utils import (
check_to_quantized,
copy_python_files_from_model_cache,
get_block_names,
+ get_lm_head_name,
get_module,
set_amax_for_all_moe_layers,
set_module,
@@ -120,6 +123,77 @@ def pack_layer(name, model, device=None):
release_layer_safely(layer)
+def _get_scheme(bits, data_type):
+ """Determine the compressed-tensors format string for a given data type and bit-width."""
+ if is_mx_fp(data_type):
+ return "MXFP4" if bits == 4 else "MXFP8"
+ if is_nv_fp(data_type):
+ return "NVFP4"
+ return None
+
+
+def _get_group_format(bits, data_type):
+ """Determine the compressed-tensors format string for a given data type and bit-width."""
+ if is_mx_fp(data_type):
+ return "mxfp4-pack-quantized" if bits == 4 else "mxfp8-quantized"
+ if is_nv_fp(data_type):
+ return "nvfp4-pack-quantized"
+ return "float-quantized"
+
+
+def _build_mixed_fp_quantization_config(scheme_groups, layer_config, ignore, global_bits, global_data_type):
+ """Build a quantization config dict for mixed-precision scenarios.
+
+ Creates multiple config groups with per-group format strings using compressed_tensors
+ QuantizationScheme objects. Override groups (non-default schemes with specific layer
+ targets) come first, and the default group (matching global bits/data_type with
+ targets=["Linear"]) comes last. Top-level format is set to "mixed-precision".
+
+ Args:
+ scheme_groups: dict mapping (bits, data_type) -> list of layer names
+ layer_config: per-layer quantization configs
+ ignore: list of layers/patterns to ignore
+ global_bits: global quantization bit-width
+ global_data_type: global quantization data type
+ Returns:
+ quantization_config dict
+ """
+ global_key = (global_bits, global_data_type)
+
+ # Override groups first, default group last
+ override_groups = []
+ default_group = None
+ for key, layer_names in scheme_groups.items():
+ if key == global_key:
+ default_group = (key, layer_names)
+ else:
+ override_groups.append((key, layer_names))
+ ordered = override_groups + ([default_group] if default_group else [])
+
+ config_groups = {}
+ group_formats = {}
+ for idx, ((lbits, ldata_type), layer_names) in enumerate(ordered):
+ group_name = f"group_{idx}"
+ scheme = _get_scheme(lbits, ldata_type)
+ tmp_quantization_config = initialize_quantization(scheme=scheme)
+ tmp_quantization_config = tmp_quantization_config.config_groups["group_0"]
+ is_default = (lbits, ldata_type) == global_key
+ tmp_quantization_config.targets = ["Linear"] if is_default else layer_names
+ config_groups[group_name] = tmp_quantization_config
+ group_formats[group_name] = _get_group_format(lbits, ldata_type)
+
+ quantization_config = initialize_quantization(scheme=None, config_groups=config_groups, ignore=ignore)
+ quantization_config = quantization_config.to_dict()
+
+ # Set per-group format and top-level mixed-precision format
+ quantization_config["format"] = "mixed-precision"
+ for group_name, fmt in group_formats.items():
+ quantization_config["config_groups"][group_name]["format"] = fmt
+ quantization_config["provider"] = "auto-round"
+
+ return quantization_config
+
+
def save_quantized_as_fp(
output_dir: str,
model: torch.nn.Module = None,
@@ -207,27 +281,38 @@ def save_quantized_as_fp(
pass
ignore = generate_ignore_regex_list(regex_config=regex_config, layer_config=layer_config)
+ lm_head_name = get_lm_head_name(model)
+ if lm_head_name and lm_head_name not in layer_config and lm_head_name not in ignore:
+ ignore.append(lm_head_name)
# get llm-compressor format config
check_compressed_tensors_supported()
- from .config import initialize_quantization
- scheme = "NVFP4"
- quantization_config = initialize_quantization(scheme=scheme)
- setattr(quantization_config, "format", "nvfp4-pack-quantized")
- setattr(quantization_config, "ignore", ignore)
- quantization_config = quantization_config.to_dict()
- quantization_config["provider"] = "auto-round"
- if is_mx_fp(data_type): # Manually replace some parameters, as compressed-tensor is not support MXFP scheme yet
- quantization_config["config_groups"]["group_0"]["weights"]["num_bits"] = bits
- quantization_config["config_groups"]["group_0"]["input_activations"]["num_bits"] = (
- act_bits if act_bits <= 8 else bits
- )
- quantization_config["config_groups"]["group_0"]["weights"]["is_mx"] = True
- quantization_config["config_groups"]["group_0"]["input_activations"]["is_mx"] = True
- quantization_config["config_groups"]["group_0"]["weights"]["group_size"] = 32
- quantization_config["config_groups"]["group_0"]["input_activations"]["group_size"] = 32
- quantization_config["format"] = "float-quantized"
+ # Detect mixed precision by grouping quantized layers by (bits, data_type)
+ scheme_groups = {} # (bits, data_type) -> list of layer names
+ for name, cfg in layer_config.items():
+ layer_bits = cfg.get("bits", bits)
+ layer_dt = cfg.get("data_type", data_type)
+ if layer_bits > 8:
+ continue
+ key = (layer_bits, layer_dt)
+ scheme_groups.setdefault(key, []).append(name)
+
+ is_mixed = len(scheme_groups) > 1
+
+ if is_mixed:
+ quantization_config = _build_mixed_fp_quantization_config(scheme_groups, layer_config, ignore, bits, data_type)
+ else:
+ scheme = _get_scheme(bits, data_type)
+ if scheme is None:
+ raise ValueError(f"Unsupported combination of data_type={data_type} and bits={bits}.")
+
+ format = _get_group_format(bits, data_type)
+ quantization_config = initialize_quantization(scheme=scheme)
+ setattr(quantization_config, "format", format)
+ setattr(quantization_config, "ignore", ignore)
+ quantization_config = quantization_config.to_dict()
+ quantization_config["provider"] = "auto-round"
if hasattr(model, "config"):
model.config.quantization_config = quantization_config
if output_dir is None:
diff --git a/test/test_cpu/export/test_llmc_format.py b/test/test_cpu/export/test_llmc_format.py
index ebc1bd87c..47db2b0f8 100644
--- a/test/test_cpu/export/test_llmc_format.py
+++ b/test/test_cpu/export/test_llmc_format.py
@@ -3,9 +3,10 @@ import shutil
import pytest
import torch
+import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer
-from auto_round import AutoRound
+from auto_round import AutoRound, AutoScheme
from ...helpers import get_model_path, opt_name_or_path
@@ -21,12 +22,6 @@ class TestLLMC:
def teardown_class(self):
shutil.rmtree("runs", ignore_errors=True)
- @pytest.fixture(autouse=True)
- def _save_dir(self, tmp_path):
- self.save_dir = str(tmp_path / "saved")
- yield
- shutil.rmtree(self.save_dir, ignore_errors=True)
-
# remove since w8a8 not in llmcompressor format supported schemes
# def test_llmcompressor_w8a8(self):
# bits, group_size, sym, act_bits = 8, -1, True, 8
@@ -45,7 +40,7 @@ class TestLLMC:
# autoround.quantize()
# autoround.save_quantized("./saved", format="llm_compressor", inplace=True)
- def test_llmcompressor_fp8(self):
+ def test_llmcompressor_fp8(self, tmp_path):
## quantize the model
model_name = opt_name_or_path
autoround = AutoRound(
@@ -55,21 +50,21 @@ class TestLLMC:
nsamples=2,
iters=0,
)
- autoround.quantize_and_save(self.save_dir, format="llm_compressor")
+ autoround.quantize_and_save(tmp_path, format="llm_compressor")
# from vllm import LLM
- # model = LLM(self.save_dir)
+ # model = LLM(tmp_path)
# result = model.generate("Hello my name is")
# print(result)
import json
- config = json.load(open(os.path.join(self.save_dir, "config.json")))
+ config = json.load(open(os.path.join(tmp_path, "config.json")))
assert "group_0" in config["quantization_config"]["config_groups"]
assert config["quantization_config"]["config_groups"]["group_0"]["input_activations"]["num_bits"] == 8
assert config["quantization_config"]["config_groups"]["group_0"]["weights"]["strategy"] == "channel"
assert config["quantization_config"]["quant_method"] == "compressed-tensors"
- def test_autoround_llmcompressor_fp8(self):
+ def test_autoround_llmcompressor_fp8(self, tmp_path):
## quantize the model
model_name = opt_name_or_path
autoround = AutoRound(
@@ -80,13 +75,79 @@ class TestLLMC:
nsamples=2,
iters=0,
)
- autoround.quantize_and_save(self.save_dir, format="auto_round:llm_compressor")
+ autoround.quantize_and_save(tmp_path, format="auto_round:llm_compressor")
import json
- config = json.load(open(os.path.join(self.save_dir, "config.json")))
+ config = json.load(open(os.path.join(tmp_path, "config.json")))
assert "group_0" in config["quantization_config"]["config_groups"]
assert config["quantization_config"]["config_groups"]["group_0"]["input_activations"]["num_bits"] == 8
assert config["quantization_config"]["config_groups"]["group_0"]["weights"]["strategy"] == "tensor"
assert config["quantization_config"]["config_groups"]["group_0"]["input_activations"]["strategy"] == "tensor"
assert config["quantization_config"]["quant_method"] == "compressed-tensors"
+
+ def test_mxfp8_llmcompressor_format(self, tiny_opt_model_path, tmp_path):
+ scheme = "mxfp8"
+ ar = AutoRound(
+ model=tiny_opt_model_path,
+ iters=0,
+ disable_opt_rtn=True,
+ scheme=scheme,
+ )
+ compressed_model, _ = ar.quantize_and_save(output_dir=tmp_path, format="llm_compressor")
+ tmp_layer = compressed_model.model.decoder.layers[1].self_attn.q_proj
+ assert (
+ hasattr(tmp_layer, "weight_scale")
+ and hasattr(tmp_layer, "weight")
+ and tmp_layer.weight_scale.dtype is torch.uint8
+ and tmp_layer.weight_scale.shape[0] == 768
+ ), "Illegal MXFP8 packing name or data_type or shape"
+ quantization_config = transformers.AutoConfig.from_pretrained(
+ tmp_path, trust_remote_code=True
+ ).quantization_config
+ assert (
+ quantization_config["format"] == "mxfp8-quantized"
+ and quantization_config["config_groups"]["group_0"]["weights"]["num_bits"] == 8
+ and quantization_config["config_groups"]["group_0"]["weights"]["group_size"] == 32
+ and quantization_config["config_groups"]["group_0"]["weights"]["scale_dtype"] == "torch.uint8"
+ and quantization_config["config_groups"]["group_0"]["input_activations"]["num_bits"] == 8
+ and quantization_config["config_groups"]["group_0"]["input_activations"]["group_size"] == 32
+ and quantization_config["config_groups"]["group_0"]["input_activations"]["scale_dtype"] == "torch.uint8"
+ and quantization_config["ignore"] == ["lm_head"]
+ ), f"Invalid MXFP8 quantization configuration: {quantization_config}"
+
+ def test_mixed_precision_llmcompressor_format(self, tiny_opt_model_path, tmp_path):
+ scheme = AutoScheme(
+ avg_bits=7,
+ options=("MXFP4", "MXFP8"),
+ shared_layers=["q_proj", "k_proj", "v_proj"],
+ )
+ ar = AutoRound(
+ model=tiny_opt_model_path,
+ iters=0,
+ disable_opt_rtn=True,
+ scheme=scheme,
+ )
+ ar.quantize_and_save(output_dir=tmp_path, format="llm_compressor")
+ model = AutoModelForCausalLM.from_pretrained(tmp_path, torch_dtype="auto", trust_remote_code=True)
+ op = model.model.decoder.layers[0].fc1
+ if op.quantization_scheme.targets != ["Linear"]:
+ assert (
+ op.quantization_scheme.weights.num_bits == 8
+ and op.quantization_scheme.input_activations.num_bits == 8
+ and op.quantization_scheme.weights.group_size == 32
+ and op.quantization_scheme.input_activations.group_size == 32
+ and op.quantization_scheme.weights.scale_dtype == torch.uint8
+ and op.quantization_scheme.input_activations.scale_dtype == torch.uint8
+ ), "Illegal MXFP4 packing name or data_type or shape"
+ quantization_config = model.config.quantization_config.to_dict()
+ assert (
+ quantization_config["format"] == "mixed-precision"
+ and quantization_config["config_groups"]["group_0"]["weights"]["num_bits"] == 8
+ and quantization_config["config_groups"]["group_0"]["input_activations"]["num_bits"] == 8
+ and quantization_config["config_groups"]["group_0"]["format"] == "mxfp8-quantized"
+ and quantization_config["config_groups"]["group_1"]["weights"]["num_bits"] == 4
+ and quantization_config["config_groups"]["group_1"]["input_activations"]["num_bits"] == 4
+ and quantization_config["config_groups"]["group_1"]["format"] == "mxfp4-pack-quantized"
+ and quantization_config["ignore"] == ["lm_head"]
+ ), f"Invalid mixed precision quantization configuration: {quantization_config}"
diff --git a/test/test_cpu/quantization/test_mix_bits.py b/test/test_cpu/quantization/test_mix_bits.py
index 71a0dc1cd..1fbbfed44 100644
--- a/test/test_cpu/quantization/test_mix_bits.py
+++ b/test/test_cpu/quantization/test_mix_bits.py
@@ -44,8 +44,8 @@ class TestAutoRound:
@require_gptqmodel
def test_mixed_gptqmodel(self, dataloader):
layer_config = {
- "k_proj": {"bits": 8}, # part name
- "lm_head": {"bits": 4}, # set lm_head quant
+ "k_proj": {"scheme": "W8A16"}, # part name
+ "lm_head": {"scheme": "w4a16"}, # set lm_head quant
"fc1": {"bits": 16},
"model.decoder.layers.0.self_attn.v_proj": {"bits": 16},
"model.decoder.layers.0.self_attn.q_proj": {"bits": 8}, # full name
diff --git a/test/test_cpu/quantization/test_mxfp_nvfp.py b/test/test_cpu/quantization/test_mxfp_nvfp.py
index fb8d51c84..2f92f73f8 100644
--- a/test/test_cpu/quantization/test_mxfp_nvfp.py
+++ b/test/test_cpu/quantization/test_mxfp_nvfp.py
@@ -152,8 +152,7 @@ class TestAutoRoundFP:
quantized_model_path, trust_remote_code=True
).quantization_config
assert (
- quantization_config["format"] == "float-quantized"
- and quantization_config["config_groups"]["group_0"]["weights"]["is_mx"] is True
+ quantization_config["format"] == "mxfp4-pack-quantized"
and quantization_config["config_groups"]["group_0"]["weights"]["num_bits"] == 4
), f"Invalid MXFP4 quantization configuration: {quantization_config}"
@@ -191,8 +190,7 @@ class TestAutoRoundFP:
quantized_model_path, trust_remote_code=True
).quantization_config
assert (
- quantization_config["format"] == "float-quantized"
- and quantization_config["config_groups"]["group_0"]["weights"]["is_mx"] is True
+ quantization_config["format"] == "mxfp4-pack-quantized"
and quantization_config["config_groups"]["group_0"]["weights"]["num_bits"] == 4
), f"Invalid MXFP4 quantization configuration: {quantization_config}"
@@ -222,8 +220,7 @@ class TestAutoRoundFP:
quantized_model_path, trust_remote_code=True
).quantization_config
assert (
- quantization_config["format"] == "float-quantized"
- and quantization_config["config_groups"]["group_0"]["weights"]["is_mx"] is True
+ quantization_config["format"] == "mxfp8-quantized"
and quantization_config["config_groups"]["group_0"]["weights"]["num_bits"] == 8
), f"Invalid MXFP8 quantization configuration: {quantization_config}"
folder_size_gb = _get_folder_size(quantized_model_path)
diff --git a/test/test_cuda/export/test_llmc_format.py b/test/test_cuda/export/test_llmc_format.py
index fe2dddc6f..232bbb94b 100644
--- a/test/test_cuda/export/test_llmc_format.py
+++ b/test/test_cuda/export/test_llmc_format.py
@@ -53,8 +53,7 @@ class TestAutoRound:
self.save_dir, trust_remote_code=True
).quantization_config
assert (
- quantization_config["format"] == "float-quantized"
- and quantization_config["config_groups"]["group_0"]["weights"]["is_mx"] is True
+ quantization_config["format"] == "mxfp4-pack-quantized"
and quantization_config["config_groups"]["group_0"]["weights"]["num_bits"] == 4
), f"Invalid MXFP4 quantization configuration: {quantization_config}"