Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Fix packing format in quantization config and update variable assignment in tests (#1857)

Xin He64c0a5bcc77b
diff --git a/auto_round/compressors/entry.py b/auto_round/compressors/entry.py
index 8236ca89c..73f3f26fe 100644
--- a/auto_round/compressors/entry.py
+++ b/auto_round/compressors/entry.py
@@ -138,29 +138,6 @@ def is_gguf_k_target(value) -> bool:
return False
-def detect_model_type(model):
- """Detect the type of model (LLM, MLLM, or Diffusion).
-
- Args:
- model: Model instance or model path string
-
- Returns:
- str: "mllm", "diffusion", or "llm"
- """
- from auto_round.utils import is_diffusion_model, is_mllm_model
-
- # Check if it's a diffusion model first (more specific)
- if is_diffusion_model(model):
- return "diffusion"
-
- # Check if it's an MLLM
- if is_mllm_model(model):
- return "mllm"
-
- # Default to standard LLM
- return "llm"
-
-
class AutoRound(object):
# Mapping from string alias to config class (and optional defaults override).
_CONFIG_ALIASES: dict[str, type] = {
@@ -251,6 +228,8 @@ class AutoRound(object):
)
# Detect model type to determine if we need special compressor
+ from auto_round.utils.model import detect_model_type
+
model_type = detect_model_type(model)
# If the user explicitly passes processor/image_processor, treat as MLLM even if
diff --git a/auto_round/compressors/model_free.py b/auto_round/compressors/model_free.py
index d847fda65..bd9df7244 100644
--- a/auto_round/compressors/model_free.py
+++ b/auto_round/compressors/model_free.py
@@ -722,10 +722,13 @@ def _build_quantization_config(
from auto_round.version import __version__
scheme_keys = [f.name for f in fields(QuantizationScheme)]
+ # vllm only support auto_round:auto_gptq, but transformers cannot load it correctly when sym=False.
+ # So we keep auto_round for asymmetric quantization to maintain compatibility with both.
+ packing_format = "auto_round:auto_gptq" if default_scheme.get("sym", True) else "auto_round"
qconfig = {
"quant_method": "auto-round",
- "packing_format": "auto_round:auto_gptq",
+ "packing_format": packing_format,
"bits": default_scheme["bits"],
"group_size": default_scheme["group_size"],
"sym": default_scheme.get("sym", True),
diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.py
index b31202c22..1ef4e5720 100644
--- a/auto_round/compressors/utils.py
+++ b/auto_round/compressors/utils.py
@@ -27,7 +27,13 @@ from torch.amp import autocast
from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, GGUF_CONFIG, GGUF_INNER_CONFIG, QK_K, ModelType
from auto_round.logger import logger
-from auto_round.utils import check_to_quantized, get_layer_names_in_block, get_module, to_standard_regex
+from auto_round.utils import (
+ check_to_quantized,
+ compress_layer_names,
+ get_layer_names_in_block,
+ get_module,
+ to_standard_regex,
+)
if TYPE_CHECKING:
from auto_round.schemes import QuantizationScheme
@@ -1059,7 +1065,8 @@ def get_fp_layer_names(model: torch.nn.Module, ignore_layers: str):
if fp_layer in name:
not_to_quantized_layers.append(name)
not_to_quantized_layers.extend(ignore_layers) # keep regex name for later use
- logger.trace(f"not_to_quantized_layers: {not_to_quantized_layers}")
+ if not_to_quantized_layers:
+ logger.info(f"Ignored layers: {compress_layer_names(not_to_quantized_layers)}")
return not_to_quantized_layers
diff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.py
index 2288dcafc..0223d132a 100644
--- a/auto_round/eval/evaluation.py
+++ b/auto_round/eval/evaluation.py
@@ -355,7 +355,7 @@ def evaluate_with_model_path(eval_folder, device_str, autoround, args):
batch_size=args.eval_bs,
limit=args.limit,
eval_model_dtype=get_model_dtype(args.eval_model_dtype, "auto"),
- mllm=autoround.mllm,
+ mllm=getattr(autoround, "mllm", False),
add_bos_token=args.add_bos_token,
)
else:
@@ -372,7 +372,7 @@ def evaluate_with_model_path(eval_folder, device_str, autoround, args):
model_args += f",add_bos_token={args.add_bos_token}"
# Choose evaluation method based on model type
- if autoround.mllm:
+ if getattr(autoround, "mllm", False):
model_type = "hf-multimodal"
eval_bs = args.eval_bs
if eval_bs is None or eval_bs == "auto":
@@ -413,6 +413,21 @@ def run_model_evaluation(model, tokenizer, autoround, folders, formats, device_s
if model is not None:
model.eval()
+ # Detect model type for compatibility with evaluation code
+ from auto_round.utils.model import detect_model_type
+
+ # Check if evaluation Compressoris needed for language models
+ if isinstance(folders, list):
+ eval_folder = folders[-1] if folders else None
+ else:
+ eval_folder = folders
+ # set model_type for ModelFreeCompressor.
+ model_type = detect_model_type(eval_folder)
+ if hasattr(autoround, "model_context") and model_type in ("mllm", "diffusion"):
+ setattr(autoround.model_context, f"is_{model_type}", True)
+ else:
+ setattr(autoround, model_type, True)
+
# Handle diffusion models separately
if getattr(autoround, "diffusion", False):
prompt = getattr(args, "prompt", None)
@@ -422,11 +437,6 @@ def run_model_evaluation(model, tokenizer, autoround, folders, formats, device_s
evaluate_diffusion_model(args, autoround=autoround, model=model)
return
- # Check if evaluation Compressoris needed for language models
- if isinstance(folders, list):
- eval_folder = folders[-1] if folders else None
- else:
- eval_folder = folders
if args.tasks is None or args.tasks == "" or eval_folder is None:
return
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.py
index a8c214e50..58cba9863 100644
--- a/auto_round/utils/model.py
+++ b/auto_round/utils/model.py
@@ -1059,6 +1059,27 @@ def is_diffusion_model(model_or_path: Union[str, object], trust_remote_code: boo
return False
+def detect_model_type(model):
+ """Detect the type of model (LLM, MLLM, or Diffusion).
+
+ Args:
+ model: Model instance or model path string
+
+ Returns:
+ str: "mllm", "diffusion", or "llm"
+ """
+ # Check if it's a diffusion model first (more specific)
+ if is_diffusion_model(model):
+ return "diffusion"
+
+ # Check if it's an MLLM
+ if is_mllm_model(model):
+ return "mllm"
+
+ # Default to standard LLM
+ return "llm"
+
+
def is_moe_layer(module: torch.nn.Module) -> bool:
"""Returns whether the module is an MOE layer."""
return "moe" in type(module).__name__.lower() or any(
diff --git a/test/test_cpu/quantization/test_model_free_parity.py b/test/test_cpu/quantization/test_model_free_parity.py
index 008589552..787c7a627 100644
--- a/test/test_cpu/quantization/test_model_free_parity.py
+++ b/test/test_cpu/quantization/test_model_free_parity.py
@@ -16,7 +16,7 @@
``AutoRound(iters=0, disable_opt_rtn=True, disable_model_free=True)`` flow.
Both code paths perform RTN integer weight-only quantization in the
-``auto_round:auto_gptq`` packing format. These tests assert that:
+``auto_round`` packing format. These tests assert that:
1. The ``quantization_config`` keys ``bits``, ``group_size``, ``sym``,
``data_type`` (family), ``quant_method``, ``packing_format`` agree.
@@ -77,14 +77,14 @@ _PARITY_SCHEMES = [
("W2A16G32", "W2A16G32", {"bits": 2, "group_size": 32, "sym": True}),
("W8A16", "W8A16", {"bits": 8, "group_size": 128, "sym": True}),
# Asymmetric (sym=False) variants for non-4-bit: both the model-free path and
- # the regular path use ``auto_round:auto_gptq`` packing, so bit-exact parity holds.
+ # the regular path use ``auto_round`` packing, so bit-exact parity holds.
("W4A16_ASYM", "W4A16", {"bits": 4, "group_size": 128, "sym": False}),
("W2A16_ASYM", "W2A16", {"bits": 2, "group_size": 128, "sym": False}),
("W2A16G64_ASYM", "W2A16G64", {"bits": 2, "group_size": 64, "sym": False}),
("W8A16_ASYM", "W8A16", {"bits": 8, "group_size": 128, "sym": False}),
# Note: W4A16 asym is excluded because the regular path uses
# ``auto_round:auto_awq`` packing for 4-bit asym, which differs from
- # model-free's ``auto_round:auto_gptq``.
+ # model-free's ``auto_round``.
]
@@ -180,9 +180,6 @@ def test_parity_model_free_vs_disable_opt_rtn(tmp_path, tiny_opt_model_path, sch
assert qc_a[key] == qc_b[key], f"qconfig[{key}] differs: model_free={qc_a[key]} regular={qc_b[key]}"
assert "int" in qc_a.get("data_type", "")
assert "int" in qc_b.get("data_type", "")
- assert (
- qc_a["packing_format"] == qc_b["packing_format"]
- ), f"packing_format differs: {qc_a['packing_format']} vs {qc_b['packing_format']}"
# ---- 2. quantized tensor key sets agree ----
tensors_a = _load_all_keys_and_tensors(out_a)
diff --git a/test/test_cpu/utils/test_cli_usage.py b/test/test_cpu/utils/test_cli_usage.py
index f26b98f7f..64308cf43 100644
--- a/test/test_cpu/utils/test_cli_usage.py
+++ b/test/test_cpu/utils/test_cli_usage.py
@@ -27,51 +27,59 @@ class TestAutoRoundCmd:
def test_auto_round_cmd(self, tiny_opt_model_path, tiny_qwen_vl_model_path):
python_path = sys.executable
-
- # Test llm script
res = os.system(f"PYTHONPATH='{AUTO_ROUND_PATH}:$PYTHONPATH' {python_path} -m auto_round -h")
if res > 0 or res == -1:
assert False, "cmd line test fail, please have a check"
+ def test_auto_round_cmd2(self, tiny_opt_model_path, tiny_qwen_vl_model_path):
+ python_path = sys.executable
res = os.system(
- f"PYTHONPATH='{AUTO_ROUND_PATH}:$PYTHONPATH' {python_path} -m auto_round --model {tiny_opt_model_path} --seqlen 32 --iter 2 --nsamples 1 --format auto_gptq,auto_round --output_dir {self.save_dir} --tasks piqa"
+ f"PYTHONPATH='{AUTO_ROUND_PATH}:$PYTHONPATH' {python_path} -m auto_round --model {tiny_opt_model_path} --seqlen 32 --iter 2 --nsamples 1 --format auto_gptq,auto_round --output_dir {self.save_dir} --tasks piqa --limit 2"
)
if res > 0 or res == -1:
assert False, "cmd line test fail, please have a check"
+ def test_auto_round_cmd3(self, tiny_opt_model_path, tiny_qwen_vl_model_path):
+ python_path = sys.executable
res = os.system(
- f"PYTHONPATH='{AUTO_ROUND_PATH}:$PYTHONPATH' {python_path} -m auto_round --model {tiny_opt_model_path} --seqlen 8 --iter 1 --nsamples 1 --eval_task_by_task --tasks openbookqa --bs 32"
+ f"PYTHONPATH='{AUTO_ROUND_PATH}:$PYTHONPATH' {python_path} -m auto_round --model {tiny_opt_model_path} --seqlen 8 --iter 1 --nsamples 1 --eval_task_by_task --tasks openbookqa --bs 32 --limit 2"
)
if res > 0 or res == -1:
assert False, "cmd line test fail, please have a check"
+ def test_auto_round_cmd4(self, tiny_opt_model_path, tiny_qwen_vl_model_path):
+ python_path = sys.executable
res = os.system(
- f"PYTHONPATH='{AUTO_ROUND_PATH}:$PYTHONPATH' {python_path} -c 'from auto_round.__main__ import run_light; run_light()' --seqlen 8 --iter 2 --nsamples 8 --output_dir {self.save_dir} --tasks lambada_openai"
+ f"PYTHONPATH='{AUTO_ROUND_PATH}:$PYTHONPATH' {python_path} -c 'from auto_round.__main__ import run_light; run_light()' --seqlen 8 --iter 2 --nsamples 8 --output_dir {self.save_dir} --tasks lambada_openai --limit 2"
)
if res > 0 or res == -1:
assert False, "cmd line test fail, please have a check"
- # test mllm script
-
- # test auto_round_mllm --eval help
+ def test_auto_round_cmd5(self, tiny_opt_model_path, tiny_qwen_vl_model_path):
+ python_path = sys.executable
res = os.system(f"PYTHONPATH='{AUTO_ROUND_PATH}:$PYTHONPATH' {python_path} -m auto_round --eval -h")
if res > 0 or res == -1:
assert False, "cmd line test fail, please have a check"
- # test auto_round_mllm --lmms help
+ def test_auto_round_cmd6(self, tiny_opt_model_path, tiny_qwen_vl_model_path):
+ python_path = sys.executable
res = os.system(f"PYTHONPATH='{AUTO_ROUND_PATH}:$PYTHONPATH' {python_path} -m auto_round --eval --lmms -h")
if res > 0 or res == -1:
assert False, "cmd line test fail, please have a check"
+ def test_auto_round_cmd7(self, tiny_opt_model_path, tiny_qwen_vl_model_path):
+ python_path = sys.executable
res = os.system(
f"PYTHONPATH='{AUTO_ROUND_PATH}:$PYTHONPATH' {python_path} -m auto_round --mllm --model {tiny_qwen_vl_model_path} --iter 2 --nsamples 2 --seqlen 32 --format auto_round --output_dir {self.save_dir}"
)
if res > 0 or res == -1:
assert False, "cmd line test fail, please have a check"
+ def test_auto_round_cmd8(self, tiny_opt_model_path, tiny_qwen_vl_model_path):
+ python_path = sys.executable
res = os.system(
f"PYTHONPATH='{AUTO_ROUND_PATH}:$PYTHONPATH' {python_path} -m auto_round --mllm --iter 2 --nsamples 2 --model {tiny_qwen_vl_model_path} --seqlen 32 --format auto_round"
- f" --quant_nontext_module --output_dir {self.save_dir} "
+ f" --quant_nontext_module --output_dir {self.save_dir}"
)
if res > 0 or res == -1:
assert False, "cmd line test fail, please have a check"
diff --git a/test/test_cuda/export/test_auto_round_format.py b/test/test_cuda/export/test_auto_round_format.py
index 0e645ae77..d2b2f6814 100644
--- a/test/test_cuda/export/test_auto_round_format.py
+++ b/test/test_cuda/export/test_auto_round_format.py
@@ -142,7 +142,9 @@ class TestAutoRound:
seqlen=2,
)
quantized_model_path = self.save_dir
- compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="fp8")
+ compressed_model, quantized_model_path = autoround.quantize_and_save(
+ output_dir=quantized_model_path, format="fp8"
+ )
tmp_layer = compressed_model.model.layers[1].self_attn.q_proj
assert hasattr(tmp_layer, "weight_scale_inv")
assert tmp_layer.weight.dtype is torch.float8_e4m3fn