PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Fix packing format in quantization config and update variable assignment in tests (#1857)
Xin He64c0a5bcc77b
diff --git a/auto_round/compressors/entry.py b/auto_round/compressors/entry.pyindex 8236ca89c..73f3f26fe 100644--- a/auto_round/compressors/entry.py+++ b/auto_round/compressors/entry.py@@ -138,29 +138,6 @@ def is_gguf_k_target(value) -> bool:return False-def detect_model_type(model):- """Detect the type of model (LLM, MLLM, or Diffusion).-- Args:- model: Model instance or model path string-- Returns:- str: "mllm", "diffusion", or "llm"- """- from auto_round.utils import is_diffusion_model, is_mllm_model-- # Check if it's a diffusion model first (more specific)- if is_diffusion_model(model):- return "diffusion"-- # Check if it's an MLLM- if is_mllm_model(model):- return "mllm"-- # Default to standard LLM- return "llm"--class AutoRound(object):# Mapping from string alias to config class (and optional defaults override)._CONFIG_ALIASES: dict[str, type] = {@@ -251,6 +228,8 @@ class AutoRound(object):)# Detect model type to determine if we need special compressor+ from auto_round.utils.model import detect_model_type+model_type = detect_model_type(model)# If the user explicitly passes processor/image_processor, treat as MLLM even ifdiff --git a/auto_round/compressors/model_free.py b/auto_round/compressors/model_free.pyindex d847fda65..bd9df7244 100644--- a/auto_round/compressors/model_free.py+++ b/auto_round/compressors/model_free.py@@ -722,10 +722,13 @@ def _build_quantization_config(from auto_round.version import __version__scheme_keys = [f.name for f in fields(QuantizationScheme)]+ # vllm only support auto_round:auto_gptq, but transformers cannot load it correctly when sym=False.+ # So we keep auto_round for asymmetric quantization to maintain compatibility with both.+ packing_format = "auto_round:auto_gptq" if default_scheme.get("sym", True) else "auto_round"qconfig = {"quant_method": "auto-round",- "packing_format": "auto_round:auto_gptq",+ "packing_format": packing_format,"bits": default_scheme["bits"],"group_size": default_scheme["group_size"],"sym": default_scheme.get("sym", True),diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.pyindex b31202c22..1ef4e5720 100644--- a/auto_round/compressors/utils.py+++ b/auto_round/compressors/utils.py@@ -27,7 +27,13 @@ from torch.amp import autocastfrom auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, GGUF_CONFIG, GGUF_INNER_CONFIG, QK_K, ModelTypefrom auto_round.logger import logger-from auto_round.utils import check_to_quantized, get_layer_names_in_block, get_module, to_standard_regex+from auto_round.utils import (+ check_to_quantized,+ compress_layer_names,+ get_layer_names_in_block,+ get_module,+ to_standard_regex,+)if TYPE_CHECKING:from auto_round.schemes import QuantizationScheme@@ -1059,7 +1065,8 @@ def get_fp_layer_names(model: torch.nn.Module, ignore_layers: str):if fp_layer in name:not_to_quantized_layers.append(name)not_to_quantized_layers.extend(ignore_layers) # keep regex name for later use- logger.trace(f"not_to_quantized_layers: {not_to_quantized_layers}")+ if not_to_quantized_layers:+ logger.info(f"Ignored layers: {compress_layer_names(not_to_quantized_layers)}")return not_to_quantized_layersdiff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.pyindex 2288dcafc..0223d132a 100644--- a/auto_round/eval/evaluation.py+++ b/auto_round/eval/evaluation.py@@ -355,7 +355,7 @@ def evaluate_with_model_path(eval_folder, device_str, autoround, args):batch_size=args.eval_bs,limit=args.limit,eval_model_dtype=get_model_dtype(args.eval_model_dtype, "auto"),- mllm=autoround.mllm,+ mllm=getattr(autoround, "mllm", False),add_bos_token=args.add_bos_token,)else:@@ -372,7 +372,7 @@ def evaluate_with_model_path(eval_folder, device_str, autoround, args):model_args += f",add_bos_token={args.add_bos_token}"# Choose evaluation method based on model type- if autoround.mllm:+ if getattr(autoround, "mllm", False):model_type = "hf-multimodal"eval_bs = args.eval_bsif eval_bs is None or eval_bs == "auto":@@ -413,6 +413,21 @@ def run_model_evaluation(model, tokenizer, autoround, folders, formats, device_sif model is not None:model.eval()+ # Detect model type for compatibility with evaluation code+ from auto_round.utils.model import detect_model_type++ # Check if evaluation Compressoris needed for language models+ if isinstance(folders, list):+ eval_folder = folders[-1] if folders else None+ else:+ eval_folder = folders+ # set model_type for ModelFreeCompressor.+ model_type = detect_model_type(eval_folder)+ if hasattr(autoround, "model_context") and model_type in ("mllm", "diffusion"):+ setattr(autoround.model_context, f"is_{model_type}", True)+ else:+ setattr(autoround, model_type, True)+# Handle diffusion models separatelyif getattr(autoround, "diffusion", False):prompt = getattr(args, "prompt", None)@@ -422,11 +437,6 @@ def run_model_evaluation(model, tokenizer, autoround, folders, formats, device_sevaluate_diffusion_model(args, autoround=autoround, model=model)return- # Check if evaluation Compressoris needed for language models- if isinstance(folders, list):- eval_folder = folders[-1] if folders else None- else:- eval_folder = foldersif args.tasks is None or args.tasks == "" or eval_folder is None:returndiff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex a8c214e50..58cba9863 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -1059,6 +1059,27 @@ def is_diffusion_model(model_or_path: Union[str, object], trust_remote_code: booreturn False+def detect_model_type(model):+ """Detect the type of model (LLM, MLLM, or Diffusion).++ Args:+ model: Model instance or model path string++ Returns:+ str: "mllm", "diffusion", or "llm"+ """+ # Check if it's a diffusion model first (more specific)+ if is_diffusion_model(model):+ return "diffusion"++ # Check if it's an MLLM+ if is_mllm_model(model):+ return "mllm"++ # Default to standard LLM+ return "llm"++def is_moe_layer(module: torch.nn.Module) -> bool:"""Returns whether the module is an MOE layer."""return "moe" in type(module).__name__.lower() or any(diff --git a/test/test_cpu/quantization/test_model_free_parity.py b/test/test_cpu/quantization/test_model_free_parity.pyindex 008589552..787c7a627 100644--- a/test/test_cpu/quantization/test_model_free_parity.py+++ b/test/test_cpu/quantization/test_model_free_parity.py@@ -16,7 +16,7 @@``AutoRound(iters=0, disable_opt_rtn=True, disable_model_free=True)`` flow.Both code paths perform RTN integer weight-only quantization in the-``auto_round:auto_gptq`` packing format. These tests assert that:+``auto_round`` packing format. These tests assert that:1. The ``quantization_config`` keys ``bits``, ``group_size``, ``sym``,``data_type`` (family), ``quant_method``, ``packing_format`` agree.@@ -77,14 +77,14 @@ _PARITY_SCHEMES = [("W2A16G32", "W2A16G32", {"bits": 2, "group_size": 32, "sym": True}),("W8A16", "W8A16", {"bits": 8, "group_size": 128, "sym": True}),# Asymmetric (sym=False) variants for non-4-bit: both the model-free path and- # the regular path use ``auto_round:auto_gptq`` packing, so bit-exact parity holds.+ # the regular path use ``auto_round`` packing, so bit-exact parity holds.("W4A16_ASYM", "W4A16", {"bits": 4, "group_size": 128, "sym": False}),("W2A16_ASYM", "W2A16", {"bits": 2, "group_size": 128, "sym": False}),("W2A16G64_ASYM", "W2A16G64", {"bits": 2, "group_size": 64, "sym": False}),("W8A16_ASYM", "W8A16", {"bits": 8, "group_size": 128, "sym": False}),# Note: W4A16 asym is excluded because the regular path uses# ``auto_round:auto_awq`` packing for 4-bit asym, which differs from- # model-free's ``auto_round:auto_gptq``.+ # model-free's ``auto_round``.]@@ -180,9 +180,6 @@ def test_parity_model_free_vs_disable_opt_rtn(tmp_path, tiny_opt_model_path, schassert qc_a[key] == qc_b[key], f"qconfig[{key}] differs: model_free={qc_a[key]} regular={qc_b[key]}"assert "int" in qc_a.get("data_type", "")assert "int" in qc_b.get("data_type", "")- assert (- qc_a["packing_format"] == qc_b["packing_format"]- ), f"packing_format differs: {qc_a['packing_format']} vs {qc_b['packing_format']}"# ---- 2. quantized tensor key sets agree ----tensors_a = _load_all_keys_and_tensors(out_a)diff --git a/test/test_cpu/utils/test_cli_usage.py b/test/test_cpu/utils/test_cli_usage.pyindex f26b98f7f..64308cf43 100644--- a/test/test_cpu/utils/test_cli_usage.py+++ b/test/test_cpu/utils/test_cli_usage.py@@ -27,51 +27,59 @@ class TestAutoRoundCmd:def test_auto_round_cmd(self, tiny_opt_model_path, tiny_qwen_vl_model_path):python_path = sys.executable-- # Test llm scriptres = os.system(f"PYTHONPATH='{AUTO_ROUND_PATH}:$PYTHONPATH' {python_path} -m auto_round -h")if res > 0 or res == -1:assert False, "cmd line test fail, please have a check"+ def test_auto_round_cmd2(self, tiny_opt_model_path, tiny_qwen_vl_model_path):+ python_path = sys.executableres = os.system(- f"PYTHONPATH='{AUTO_ROUND_PATH}:$PYTHONPATH' {python_path} -m auto_round --model {tiny_opt_model_path} --seqlen 32 --iter 2 --nsamples 1 --format auto_gptq,auto_round --output_dir {self.save_dir} --tasks piqa"+ f"PYTHONPATH='{AUTO_ROUND_PATH}:$PYTHONPATH' {python_path} -m auto_round --model {tiny_opt_model_path} --seqlen 32 --iter 2 --nsamples 1 --format auto_gptq,auto_round --output_dir {self.save_dir} --tasks piqa --limit 2")if res > 0 or res == -1:assert False, "cmd line test fail, please have a check"+ def test_auto_round_cmd3(self, tiny_opt_model_path, tiny_qwen_vl_model_path):+ python_path = sys.executableres = os.system(- f"PYTHONPATH='{AUTO_ROUND_PATH}:$PYTHONPATH' {python_path} -m auto_round --model {tiny_opt_model_path} --seqlen 8 --iter 1 --nsamples 1 --eval_task_by_task --tasks openbookqa --bs 32"+ f"PYTHONPATH='{AUTO_ROUND_PATH}:$PYTHONPATH' {python_path} -m auto_round --model {tiny_opt_model_path} --seqlen 8 --iter 1 --nsamples 1 --eval_task_by_task --tasks openbookqa --bs 32 --limit 2")if res > 0 or res == -1:assert False, "cmd line test fail, please have a check"+ def test_auto_round_cmd4(self, tiny_opt_model_path, tiny_qwen_vl_model_path):+ python_path = sys.executableres = os.system(- f"PYTHONPATH='{AUTO_ROUND_PATH}:$PYTHONPATH' {python_path} -c 'from auto_round.__main__ import run_light; run_light()' --seqlen 8 --iter 2 --nsamples 8 --output_dir {self.save_dir} --tasks lambada_openai"+ f"PYTHONPATH='{AUTO_ROUND_PATH}:$PYTHONPATH' {python_path} -c 'from auto_round.__main__ import run_light; run_light()' --seqlen 8 --iter 2 --nsamples 8 --output_dir {self.save_dir} --tasks lambada_openai --limit 2")if res > 0 or res == -1:assert False, "cmd line test fail, please have a check"- # test mllm script-- # test auto_round_mllm --eval help+ def test_auto_round_cmd5(self, tiny_opt_model_path, tiny_qwen_vl_model_path):+ python_path = sys.executableres = os.system(f"PYTHONPATH='{AUTO_ROUND_PATH}:$PYTHONPATH' {python_path} -m auto_round --eval -h")if res > 0 or res == -1:assert False, "cmd line test fail, please have a check"- # test auto_round_mllm --lmms help+ def test_auto_round_cmd6(self, tiny_opt_model_path, tiny_qwen_vl_model_path):+ python_path = sys.executableres = os.system(f"PYTHONPATH='{AUTO_ROUND_PATH}:$PYTHONPATH' {python_path} -m auto_round --eval --lmms -h")if res > 0 or res == -1:assert False, "cmd line test fail, please have a check"+ def test_auto_round_cmd7(self, tiny_opt_model_path, tiny_qwen_vl_model_path):+ python_path = sys.executableres = os.system(f"PYTHONPATH='{AUTO_ROUND_PATH}:$PYTHONPATH' {python_path} -m auto_round --mllm --model {tiny_qwen_vl_model_path} --iter 2 --nsamples 2 --seqlen 32 --format auto_round --output_dir {self.save_dir}")if res > 0 or res == -1:assert False, "cmd line test fail, please have a check"+ def test_auto_round_cmd8(self, tiny_opt_model_path, tiny_qwen_vl_model_path):+ python_path = sys.executableres = os.system(f"PYTHONPATH='{AUTO_ROUND_PATH}:$PYTHONPATH' {python_path} -m auto_round --mllm --iter 2 --nsamples 2 --model {tiny_qwen_vl_model_path} --seqlen 32 --format auto_round"- f" --quant_nontext_module --output_dir {self.save_dir} "+ f" --quant_nontext_module --output_dir {self.save_dir}")if res > 0 or res == -1:assert False, "cmd line test fail, please have a check"diff --git a/test/test_cuda/export/test_auto_round_format.py b/test/test_cuda/export/test_auto_round_format.pyindex 0e645ae77..d2b2f6814 100644--- a/test/test_cuda/export/test_auto_round_format.py+++ b/test/test_cuda/export/test_auto_round_format.py@@ -142,7 +142,9 @@ class TestAutoRound:seqlen=2,)quantized_model_path = self.save_dir- compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="fp8")+ compressed_model, quantized_model_path = autoround.quantize_and_save(+ output_dir=quantized_model_path, format="fp8"+ )tmp_layer = compressed_model.model.layers[1].self_attn.q_projassert hasattr(tmp_layer, "weight_scale_inv")assert tmp_layer.weight.dtype is torch.float8_e4m3fn