PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Enhance quantization configuration support for mixed precision and schemes in utils and tests (#1643)
Xin He4632b2e9f339
diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.pyindex 03d4902bf..58f64f683 100644--- a/auto_round/compressors/utils.py+++ b/auto_round/compressors/utils.py@@ -321,6 +321,12 @@ def set_layer_config(elif isinstance(item, QuantizationScheme):config = asdict(item)elif isinstance(item, dict):+ # Support "scheme" key inside dict: resolve the preset and merge overrides+ if "scheme" in item:+ scheme_name = item.pop("scheme")+ base = asdict(preset_name_to_scheme(scheme_name.upper()))+ base.update(item)+ item = baseinvalid = set(item) - set(scheme_keys + ("fixed_by_user", "scale_dtype"))if invalid:raise ValueError(diff --git a/auto_round/export/export_to_llmcompressor/config.py b/auto_round/export/export_to_llmcompressor/config.pyindex 6d6513d80..546b9addb 100644--- a/auto_round/export/export_to_llmcompressor/config.py+++ b/auto_round/export/export_to_llmcompressor/config.py@@ -63,13 +63,16 @@ def initialize_quantization(scheme, targets=["Linear"], config_groups=None, kv_cconfig_groups = config_groupskv_cache_scheme = kv_cache_schemeignore = ignore+ using_mxfp4_for_mxfp8 = Falsecheck_compressed_tensors_supported()if scheme is not None and config_groups is not None:raise ValueError("Please specify either `scheme` or `config_groups`")if scheme is not None:# takes precedence over config_groups-+ if not is_preset_scheme(scheme) and scheme.lower() == "mxfp8":+ scheme = "mxfp4"+ using_mxfp4_for_mxfp8 = True # mxfp8 is not supported.if isinstance(scheme, str) and is_preset_scheme(scheme):# attach targets to schemescheme = {scheme: targets}@@ -80,6 +83,10 @@ def initialize_quantization(scheme, targets=["Linear"], config_groups=None, kv_cscheme = preset_name_to_scheme(key, scheme[key])else:scheme = QuantizationScheme.model_validate({"targets": scheme[key], **scheme})+ if using_mxfp4_for_mxfp8:+ scheme.weights.num_bits = 8+ scheme.input_activations.num_bits = 8+ scheme.format = "mxfp8-quantized"group_name = f"group_{idx}"config_groups[group_name] = schemediff --git a/auto_round/export/export_to_llmcompressor/export_to_fp.py b/auto_round/export/export_to_llmcompressor/export_to_fp.pyindex 05c01ee3a..165903396 100644--- a/auto_round/export/export_to_llmcompressor/export_to_fp.py+++ b/auto_round/export/export_to_llmcompressor/export_to_fp.py@@ -16,6 +16,7 @@ import copyimport inspectimport jsonimport os+import sysfrom concurrent.futures import ThreadPoolExecutorfrom typing import Callable, Union@@ -27,6 +28,7 @@ from tqdm import tqdmfrom auto_round.compressors.utils import is_mx_fp, is_nv_fpfrom auto_round.export.export_to_autoround.qlinear_fp import QuantLinear+from auto_round.export.export_to_llmcompressor.config import initialize_quantizationfrom auto_round.export.export_to_llmcompressor.utils import generate_ignore_regex_listfrom auto_round.export.utils import filter_quantization_config, release_layer_safely, save_modelfrom auto_round.logger import logger@@ -36,6 +38,7 @@ from auto_round.utils import (check_to_quantized,copy_python_files_from_model_cache,get_block_names,+ get_lm_head_name,get_module,set_amax_for_all_moe_layers,set_module,@@ -120,6 +123,77 @@ def pack_layer(name, model, device=None):release_layer_safely(layer)+def _get_scheme(bits, data_type):+ """Determine the compressed-tensors format string for a given data type and bit-width."""+ if is_mx_fp(data_type):+ return "MXFP4" if bits == 4 else "MXFP8"+ if is_nv_fp(data_type):+ return "NVFP4"+ return None+++def _get_group_format(bits, data_type):+ """Determine the compressed-tensors format string for a given data type and bit-width."""+ if is_mx_fp(data_type):+ return "mxfp4-pack-quantized" if bits == 4 else "mxfp8-quantized"+ if is_nv_fp(data_type):+ return "nvfp4-pack-quantized"+ return "float-quantized"+++def _build_mixed_fp_quantization_config(scheme_groups, layer_config, ignore, global_bits, global_data_type):+ """Build a quantization config dict for mixed-precision scenarios.++ Creates multiple config groups with per-group format strings using compressed_tensors+ QuantizationScheme objects. Override groups (non-default schemes with specific layer+ targets) come first, and the default group (matching global bits/data_type with+ targets=["Linear"]) comes last. Top-level format is set to "mixed-precision".++ Args:+ scheme_groups: dict mapping (bits, data_type) -> list of layer names+ layer_config: per-layer quantization configs+ ignore: list of layers/patterns to ignore+ global_bits: global quantization bit-width+ global_data_type: global quantization data type+ Returns:+ quantization_config dict+ """+ global_key = (global_bits, global_data_type)++ # Override groups first, default group last+ override_groups = []+ default_group = None+ for key, layer_names in scheme_groups.items():+ if key == global_key:+ default_group = (key, layer_names)+ else:+ override_groups.append((key, layer_names))+ ordered = override_groups + ([default_group] if default_group else [])++ config_groups = {}+ group_formats = {}+ for idx, ((lbits, ldata_type), layer_names) in enumerate(ordered):+ group_name = f"group_{idx}"+ scheme = _get_scheme(lbits, ldata_type)+ tmp_quantization_config = initialize_quantization(scheme=scheme)+ tmp_quantization_config = tmp_quantization_config.config_groups["group_0"]+ is_default = (lbits, ldata_type) == global_key+ tmp_quantization_config.targets = ["Linear"] if is_default else layer_names+ config_groups[group_name] = tmp_quantization_config+ group_formats[group_name] = _get_group_format(lbits, ldata_type)++ quantization_config = initialize_quantization(scheme=None, config_groups=config_groups, ignore=ignore)+ quantization_config = quantization_config.to_dict()++ # Set per-group format and top-level mixed-precision format+ quantization_config["format"] = "mixed-precision"+ for group_name, fmt in group_formats.items():+ quantization_config["config_groups"][group_name]["format"] = fmt+ quantization_config["provider"] = "auto-round"++ return quantization_config++def save_quantized_as_fp(output_dir: str,model: torch.nn.Module = None,@@ -207,27 +281,38 @@ def save_quantized_as_fp(passignore = generate_ignore_regex_list(regex_config=regex_config, layer_config=layer_config)+ lm_head_name = get_lm_head_name(model)+ if lm_head_name and lm_head_name not in layer_config and lm_head_name not in ignore:+ ignore.append(lm_head_name)# get llm-compressor format configcheck_compressed_tensors_supported()- from .config import initialize_quantization- scheme = "NVFP4"- quantization_config = initialize_quantization(scheme=scheme)- setattr(quantization_config, "format", "nvfp4-pack-quantized")- setattr(quantization_config, "ignore", ignore)- quantization_config = quantization_config.to_dict()- quantization_config["provider"] = "auto-round"- if is_mx_fp(data_type): # Manually replace some parameters, as compressed-tensor is not support MXFP scheme yet- quantization_config["config_groups"]["group_0"]["weights"]["num_bits"] = bits- quantization_config["config_groups"]["group_0"]["input_activations"]["num_bits"] = (- act_bits if act_bits <= 8 else bits- )- quantization_config["config_groups"]["group_0"]["weights"]["is_mx"] = True- quantization_config["config_groups"]["group_0"]["input_activations"]["is_mx"] = True- quantization_config["config_groups"]["group_0"]["weights"]["group_size"] = 32- quantization_config["config_groups"]["group_0"]["input_activations"]["group_size"] = 32- quantization_config["format"] = "float-quantized"+ # Detect mixed precision by grouping quantized layers by (bits, data_type)+ scheme_groups = {} # (bits, data_type) -> list of layer names+ for name, cfg in layer_config.items():+ layer_bits = cfg.get("bits", bits)+ layer_dt = cfg.get("data_type", data_type)+ if layer_bits > 8:+ continue+ key = (layer_bits, layer_dt)+ scheme_groups.setdefault(key, []).append(name)++ is_mixed = len(scheme_groups) > 1++ if is_mixed:+ quantization_config = _build_mixed_fp_quantization_config(scheme_groups, layer_config, ignore, bits, data_type)+ else:+ scheme = _get_scheme(bits, data_type)+ if scheme is None:+ raise ValueError(f"Unsupported combination of data_type={data_type} and bits={bits}.")++ format = _get_group_format(bits, data_type)+ quantization_config = initialize_quantization(scheme=scheme)+ setattr(quantization_config, "format", format)+ setattr(quantization_config, "ignore", ignore)+ quantization_config = quantization_config.to_dict()+ quantization_config["provider"] = "auto-round"if hasattr(model, "config"):model.config.quantization_config = quantization_configif output_dir is None:diff --git a/test/test_cpu/export/test_llmc_format.py b/test/test_cpu/export/test_llmc_format.pyindex ebc1bd87c..47db2b0f8 100644--- a/test/test_cpu/export/test_llmc_format.py+++ b/test/test_cpu/export/test_llmc_format.py@@ -3,9 +3,10 @@ import shutilimport pytestimport torch+import transformersfrom transformers import AutoModelForCausalLM, AutoTokenizer-from auto_round import AutoRound+from auto_round import AutoRound, AutoSchemefrom ...helpers import get_model_path, opt_name_or_path@@ -21,12 +22,6 @@ class TestLLMC:def teardown_class(self):shutil.rmtree("runs", ignore_errors=True)- @pytest.fixture(autouse=True)- def _save_dir(self, tmp_path):- self.save_dir = str(tmp_path / "saved")- yield- shutil.rmtree(self.save_dir, ignore_errors=True)-# remove since w8a8 not in llmcompressor format supported schemes# def test_llmcompressor_w8a8(self):# bits, group_size, sym, act_bits = 8, -1, True, 8@@ -45,7 +40,7 @@ class TestLLMC:# autoround.quantize()# autoround.save_quantized("./saved", format="llm_compressor", inplace=True)- def test_llmcompressor_fp8(self):+ def test_llmcompressor_fp8(self, tmp_path):## quantize the modelmodel_name = opt_name_or_pathautoround = AutoRound(@@ -55,21 +50,21 @@ class TestLLMC:nsamples=2,iters=0,)- autoround.quantize_and_save(self.save_dir, format="llm_compressor")+ autoround.quantize_and_save(tmp_path, format="llm_compressor")# from vllm import LLM- # model = LLM(self.save_dir)+ # model = LLM(tmp_path)# result = model.generate("Hello my name is")# print(result)import json- config = json.load(open(os.path.join(self.save_dir, "config.json")))+ config = json.load(open(os.path.join(tmp_path, "config.json")))assert "group_0" in config["quantization_config"]["config_groups"]assert config["quantization_config"]["config_groups"]["group_0"]["input_activations"]["num_bits"] == 8assert config["quantization_config"]["config_groups"]["group_0"]["weights"]["strategy"] == "channel"assert config["quantization_config"]["quant_method"] == "compressed-tensors"- def test_autoround_llmcompressor_fp8(self):+ def test_autoround_llmcompressor_fp8(self, tmp_path):## quantize the modelmodel_name = opt_name_or_pathautoround = AutoRound(@@ -80,13 +75,79 @@ class TestLLMC:nsamples=2,iters=0,)- autoround.quantize_and_save(self.save_dir, format="auto_round:llm_compressor")+ autoround.quantize_and_save(tmp_path, format="auto_round:llm_compressor")import json- config = json.load(open(os.path.join(self.save_dir, "config.json")))+ config = json.load(open(os.path.join(tmp_path, "config.json")))assert "group_0" in config["quantization_config"]["config_groups"]assert config["quantization_config"]["config_groups"]["group_0"]["input_activations"]["num_bits"] == 8assert config["quantization_config"]["config_groups"]["group_0"]["weights"]["strategy"] == "tensor"assert config["quantization_config"]["config_groups"]["group_0"]["input_activations"]["strategy"] == "tensor"assert config["quantization_config"]["quant_method"] == "compressed-tensors"++ def test_mxfp8_llmcompressor_format(self, tiny_opt_model_path, tmp_path):+ scheme = "mxfp8"+ ar = AutoRound(+ model=tiny_opt_model_path,+ iters=0,+ disable_opt_rtn=True,+ scheme=scheme,+ )+ compressed_model, _ = ar.quantize_and_save(output_dir=tmp_path, format="llm_compressor")+ tmp_layer = compressed_model.model.decoder.layers[1].self_attn.q_proj+ assert (+ hasattr(tmp_layer, "weight_scale")+ and hasattr(tmp_layer, "weight")+ and tmp_layer.weight_scale.dtype is torch.uint8+ and tmp_layer.weight_scale.shape[0] == 768+ ), "Illegal MXFP8 packing name or data_type or shape"+ quantization_config = transformers.AutoConfig.from_pretrained(+ tmp_path, trust_remote_code=True+ ).quantization_config+ assert (+ quantization_config["format"] == "mxfp8-quantized"+ and quantization_config["config_groups"]["group_0"]["weights"]["num_bits"] == 8+ and quantization_config["config_groups"]["group_0"]["weights"]["group_size"] == 32+ and quantization_config["config_groups"]["group_0"]["weights"]["scale_dtype"] == "torch.uint8"+ and quantization_config["config_groups"]["group_0"]["input_activations"]["num_bits"] == 8+ and quantization_config["config_groups"]["group_0"]["input_activations"]["group_size"] == 32+ and quantization_config["config_groups"]["group_0"]["input_activations"]["scale_dtype"] == "torch.uint8"+ and quantization_config["ignore"] == ["lm_head"]+ ), f"Invalid MXFP8 quantization configuration: {quantization_config}"++ def test_mixed_precision_llmcompressor_format(self, tiny_opt_model_path, tmp_path):+ scheme = AutoScheme(+ avg_bits=7,+ options=("MXFP4", "MXFP8"),+ shared_layers=["q_proj", "k_proj", "v_proj"],+ )+ ar = AutoRound(+ model=tiny_opt_model_path,+ iters=0,+ disable_opt_rtn=True,+ scheme=scheme,+ )+ ar.quantize_and_save(output_dir=tmp_path, format="llm_compressor")+ model = AutoModelForCausalLM.from_pretrained(tmp_path, torch_dtype="auto", trust_remote_code=True)+ op = model.model.decoder.layers[0].fc1+ if op.quantization_scheme.targets != ["Linear"]:+ assert (+ op.quantization_scheme.weights.num_bits == 8+ and op.quantization_scheme.input_activations.num_bits == 8+ and op.quantization_scheme.weights.group_size == 32+ and op.quantization_scheme.input_activations.group_size == 32+ and op.quantization_scheme.weights.scale_dtype == torch.uint8+ and op.quantization_scheme.input_activations.scale_dtype == torch.uint8+ ), "Illegal MXFP4 packing name or data_type or shape"+ quantization_config = model.config.quantization_config.to_dict()+ assert (+ quantization_config["format"] == "mixed-precision"+ and quantization_config["config_groups"]["group_0"]["weights"]["num_bits"] == 8+ and quantization_config["config_groups"]["group_0"]["input_activations"]["num_bits"] == 8+ and quantization_config["config_groups"]["group_0"]["format"] == "mxfp8-quantized"+ and quantization_config["config_groups"]["group_1"]["weights"]["num_bits"] == 4+ and quantization_config["config_groups"]["group_1"]["input_activations"]["num_bits"] == 4+ and quantization_config["config_groups"]["group_1"]["format"] == "mxfp4-pack-quantized"+ and quantization_config["ignore"] == ["lm_head"]+ ), f"Invalid mixed precision quantization configuration: {quantization_config}"diff --git a/test/test_cpu/quantization/test_mix_bits.py b/test/test_cpu/quantization/test_mix_bits.pyindex 71a0dc1cd..1fbbfed44 100644--- a/test/test_cpu/quantization/test_mix_bits.py+++ b/test/test_cpu/quantization/test_mix_bits.py@@ -44,8 +44,8 @@ class TestAutoRound:@require_gptqmodeldef test_mixed_gptqmodel(self, dataloader):layer_config = {- "k_proj": {"bits": 8}, # part name- "lm_head": {"bits": 4}, # set lm_head quant+ "k_proj": {"scheme": "W8A16"}, # part name+ "lm_head": {"scheme": "w4a16"}, # set lm_head quant"fc1": {"bits": 16},"model.decoder.layers.0.self_attn.v_proj": {"bits": 16},"model.decoder.layers.0.self_attn.q_proj": {"bits": 8}, # full namediff --git a/test/test_cpu/quantization/test_mxfp_nvfp.py b/test/test_cpu/quantization/test_mxfp_nvfp.pyindex fb8d51c84..2f92f73f8 100644--- a/test/test_cpu/quantization/test_mxfp_nvfp.py+++ b/test/test_cpu/quantization/test_mxfp_nvfp.py@@ -152,8 +152,7 @@ class TestAutoRoundFP:quantized_model_path, trust_remote_code=True).quantization_configassert (- quantization_config["format"] == "float-quantized"- and quantization_config["config_groups"]["group_0"]["weights"]["is_mx"] is True+ quantization_config["format"] == "mxfp4-pack-quantized"and quantization_config["config_groups"]["group_0"]["weights"]["num_bits"] == 4), f"Invalid MXFP4 quantization configuration: {quantization_config}"@@ -191,8 +190,7 @@ class TestAutoRoundFP:quantized_model_path, trust_remote_code=True).quantization_configassert (- quantization_config["format"] == "float-quantized"- and quantization_config["config_groups"]["group_0"]["weights"]["is_mx"] is True+ quantization_config["format"] == "mxfp4-pack-quantized"and quantization_config["config_groups"]["group_0"]["weights"]["num_bits"] == 4), f"Invalid MXFP4 quantization configuration: {quantization_config}"@@ -222,8 +220,7 @@ class TestAutoRoundFP:quantized_model_path, trust_remote_code=True).quantization_configassert (- quantization_config["format"] == "float-quantized"- and quantization_config["config_groups"]["group_0"]["weights"]["is_mx"] is True+ quantization_config["format"] == "mxfp8-quantized"and quantization_config["config_groups"]["group_0"]["weights"]["num_bits"] == 8), f"Invalid MXFP8 quantization configuration: {quantization_config}"folder_size_gb = _get_folder_size(quantized_model_path)diff --git a/test/test_cuda/export/test_llmc_format.py b/test/test_cuda/export/test_llmc_format.pyindex fe2dddc6f..232bbb94b 100644--- a/test/test_cuda/export/test_llmc_format.py+++ b/test/test_cuda/export/test_llmc_format.py@@ -53,8 +53,7 @@ class TestAutoRound:self.save_dir, trust_remote_code=True).quantization_configassert (- quantization_config["format"] == "float-quantized"- and quantization_config["config_groups"]["group_0"]["weights"]["is_mx"] is True+ quantization_config["format"] == "mxfp4-pack-quantized"and quantization_config["config_groups"]["group_0"]["weights"]["num_bits"] == 4), f"Invalid MXFP4 quantization configuration: {quantization_config}"