PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Fix CI coverage & bug grep issue (#1893)
chen, suyuea2fbad794485
diff --git a/.azure-pipelines/scripts/ut/.coverage b/.azure-pipelines/scripts/ut/.coverageindex 5be62fc0d..28b2c2c9a 100644--- a/.azure-pipelines/scripts/ut/.coverage+++ b/.azure-pipelines/scripts/ut/.coverage@@ -3,10 +3,10 @@ branch = True[paths]source =- auto_round/- /auto-round/auto_round/+ auto_round+ */_work/*/s/auto_round+ */auto-round/auto_round*/site-packages/auto_round/- */dist-packages/auto_round/[report]include =diff --git a/.azure-pipelines/scripts/ut/collect_result.py b/.azure-pipelines/scripts/ut/collect_result.pyindex 2f0a0c122..7795031bf 100644--- a/.azure-pipelines/scripts/ut/collect_result.py+++ b/.azure-pipelines/scripts/ut/collect_result.py@@ -3,7 +3,6 @@import argparseimport reimport sys-from collections import dequefrom dataclasses import dataclassfrom enum import Enum, autofrom pathlib import Path@@ -52,15 +51,20 @@ class LogAnalyzer:)ERROR_MARKERS = (- "ERROR",+ "ERROR:","== ERRORS ==",- " errors",+ " errors:","Exception","Traceback",)PASS_MARKER = " passed"+ SKIP_MARKERS = (+ " deselected",+ " skipped",+ )+def __init__(self, log_dir: Path, log_pattern: str = "*.log"):self.log_dir = Path(log_dir)self.log_pattern = log_pattern@@ -89,10 +93,10 @@ class LogAnalyzer:duration=self._extract_duration(content),)- def _read_log(self, path: Path, max_lines: int = 100) -> str:+ def _read_log(self, path: Path) -> str:try:with open(path, "r", encoding="utf-8", errors="ignore") as f:- return "".join(deque(f, maxlen=max_lines))+ return f.read()except OSError as e:print(f"Warning: Cannot read {path} - {e}", file=sys.stderr)return ""@@ -112,7 +116,9 @@ class LogAnalyzer:return TestStatus.FAILEDif self.PASS_MARKER in content:return TestStatus.PASSED- return TestStatus.NO_TESTS+ if any(marker in content for marker in self.SKIP_MARKERS):+ return TestStatus.NO_TESTS+ return TestStatus.FAILEDdef _extract_duration(self, content: str) -> str:last_match = Nonediff --git a/.azure-pipelines/scripts/ut/run_ut.sh b/.azure-pipelines/scripts/ut/run_ut.shindex b4dbb261e..0da5c2a7a 100644--- a/.azure-pipelines/scripts/ut/run_ut.sh+++ b/.azure-pipelines/scripts/ut/run_ut.sh@@ -17,14 +17,11 @@ function setup_environment() {uv pip install pytest-cov pytest-htmluv pip list- # workaround for ark test, remove auto_round_kernel_xpu- package_path=$(uv pip show auto-round-lib | grep Location:|cut -d: -f2)- rm -rf $package_path/auto_round_kernel/auto_round_kernel_xpu*# install latest gguf for ut testcd ~ || exit 1- git clone -b master --quiet --single-branch https://github.com/ggml-org/llama.cpp.git && cd llama.cpp/gguf-py && uv pip install . sentencepiece-+ git clone -b master --quiet --single-branch https://github.com/ggml-org/llama.cpp.git && cd llama.cpp/gguf-py && uv pip install .+cd /auto-round && uv pip install .export LD_LIBRARY_PATH=${HOME}/.venv/lib/:$LD_LIBRARY_PATH@@ -64,7 +61,6 @@ function check_storage_usage() {function run_unit_test() {cd /auto-round/test || exit 1- auto_round_path=$(python -c 'import auto_round; print(auto_round.__path__[0])')# Split test files into 5 partsfind ./test_cpu -name "test*.py" | grep -Ev "test_llmc|test_inc" | sort > all_tests.txt@@ -88,7 +84,7 @@ function run_unit_test() {local ut_log_name=${LOG_DIR}/unittest_${test_basename}.lognumactl --physcpubind="${NUMA_CPUSET:-0-15}" --membind="${NUMA_NODE:-0}" \- python -m pytest --cov="${auto_round_path}" --cov-report term --html=report.html --self-contained-html \+ python -m pytest --cov=auto_round --cov-report= --html=report.html --self-contained-html \--cov-report xml:coverage.xml --cov-append \-vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}echo "##[endgroup]"@@ -97,11 +93,10 @@ function run_unit_test() {function run_inc_unit_test() {echo "##[group]set up INC UT env..."- INC_PT_ONLY=1 uv pip install -r /auto-round/test/test_cpu/requirements_inc.txt+ INC_PT_ONLY=1 uv pip install -r /auto-round/test/test_cpu/requirements_inc.txt --extra-index-url https://download.pytorch.org/whl/cpuecho "##[endgroup]"cd /auto-round/test || exit 1- auto_round_path=$(python -c 'import auto_round; print(auto_round.__path__[0])')for test_file in $(find ./test_cpu -name "test_inc*.py" | sort); doecho "##[group]Running ${test_file}..."@@ -109,7 +104,7 @@ function run_inc_unit_test() {local ut_log_name=${LOG_DIR}/unittest_${test_basename}.lognumactl --physcpubind="${NUMA_CPUSET:-0-15}" --membind="${NUMA_NODE:-0}" \- python -m pytest --cov="${auto_round_path}" --cov-report term --html=report.html --self-contained-html \+ python -m pytest --cov=auto_round --cov-report= --html=report.html --self-contained-html \--cov-report xml:coverage.xml --cov-append \-vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}echo "##[endgroup]"@@ -118,13 +113,12 @@ function run_inc_unit_test() {function run_llmc_unit_test() {echo "##[group]set up LLMC UT env..."- BUILD_TYPE="nightly" uv pip install -r /auto-round/test/test_cpu/requirements_llmc.txt+ BUILD_TYPE="nightly" uv pip install -r /auto-round/test/test_cpu/requirements_llmc.txt --extra-index-url https://download.pytorch.org/whl/cpuuv pip uninstall auto-roundcd /auto-round && uv pip install .echo "##[endgroup]"cd /auto-round/test || exit 1- auto_round_path=$(python -c 'import auto_round; print(auto_round.__path__[0])')for test_file in $(find ./test_cpu -name "test_llmc*.py" | sort); doecho "##[group]Running ${test_file}..."@@ -132,7 +126,7 @@ function run_llmc_unit_test() {local ut_log_name=${LOG_DIR}/unittest_${test_basename}.lognumactl --physcpubind="${NUMA_CPUSET:-0-15}" --membind="${NUMA_NODE:-0}" \- python -m pytest --cov="${auto_round_path}" --cov-report term --html=report.html --self-contained-html \+ python -m pytest --cov=auto_round --cov-report= --html=report.html --self-contained-html \--cov-report xml:coverage.xml --cov-append \-vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}echo "##[endgroup]"diff --git a/.azure-pipelines/scripts/ut/run_ut_hpu.sh b/.azure-pipelines/scripts/ut/run_ut_hpu.shindex e8b36c1d3..4023eddb1 100644--- a/.azure-pipelines/scripts/ut/run_ut_hpu.sh+++ b/.azure-pipelines/scripts/ut/run_ut_hpu.sh@@ -34,7 +34,7 @@ function run_unit_test() {echo "##[group]Running ${test_file} in HPU lazy mode..."local ut_log_name="${LOG_DIR}/unittest_lazy_${test_basename}.log"PT_HPU_LAZY_MODE=1 pytest --cov="${auto_round_path}" \- --cov-report term --html=report.html --self-contained-html \+ --cov-report= --html=report.html --self-contained-html \--cov-report xml:coverage.xml --cov-append -vs --disable-warnings \${test_file} 2>&1 | tee ${ut_log_name}echo "##[endgroup]"@@ -42,7 +42,7 @@ function run_unit_test() {echo "##[group]Running ${test_file} in HPU compile mode..."local ut_log_name="${LOG_DIR}/unittest_compile_${test_basename}.log"PT_HPU_LAZY_MODE=0 pytest --mode compile --cov="${auto_round_path}" \- --cov-report term --html=report.html --self-contained-html \+ --cov-report= --html=report.html --self-contained-html \--cov-report xml:coverage.xml --cov-append -vs --disable-warnings \${test_file} 2>&1 | tee ${ut_log_name}echo "##[endgroup]"@@ -74,10 +74,17 @@ function collect_log() {cp .coverage "${LOG_DIR}/.coverage"}+function print_coverage() {+ echo "##[group]overall code coverage..."+ python -m coverage report+ echo "##[endgroup]"+}+function main() {setup_environmentrun_unit_testcollect_log+ print_coverageprint_summary}diff --git a/.azure-pipelines/scripts/ut/run_ut_xpu.sh b/.azure-pipelines/scripts/ut/run_ut_xpu.shindex 76f405300..5b9e784e3 100644--- a/.azure-pipelines/scripts/ut/run_ut_xpu.sh+++ b/.azure-pipelines/scripts/ut/run_ut_xpu.sh@@ -39,7 +39,7 @@ function run_unit_test() {echo "##[group]Running ark ${test_file}..."local ut_log_name="${LOG_DIR}/unittest_ark_${test_basename}.log"numactl --physcpubind="${NUMA_CPUSET:-0-27}" --membind="${NUMA_NODE:-0}" pytest --cov="${auto_round_path}" \- --cov-report term --html=report.html --self-contained-html \+ --cov-report= --html=report.html --self-contained-html \--cov-report xml:coverage.xml --cov-append -vs --disable-warnings \${test_file} 2>&1 | tee ${ut_log_name}echo "##[endgroup]"@@ -51,7 +51,7 @@ function run_unit_test() {echo "##[group]Running xpu ${test_file}..."local ut_log_name="${LOG_DIR}/unittest_xpu_${test_basename}.log"numactl --physcpubind="${NUMA_CPUSET:-0-27}" --membind="${NUMA_NODE:-0}" pytest --cov="${auto_round_path}" \- --cov-report term --html=report.html --self-contained-html \+ --cov-report= --html=report.html --self-contained-html \--cov-report xml:coverage.xml --cov-append -vs --disable-warnings \${test_file} 2>&1 | tee ${ut_log_name}echo "##[endgroup]"@@ -72,7 +72,7 @@ function run_unit_test_vllm() {echo "##[group]Running xpu vLLM ${test_file}..."local ut_log_name="${LOG_DIR}/unittest_xpu_${test_basename}.log"numactl --physcpubind="${NUMA_CPUSET:-0-27}" --membind="${NUMA_NODE:-0}" pytest --cov="${auto_round_path}" \- --cov-report term --html=report.html --self-contained-html \+ --cov-report= --html=report.html --self-contained-html \--cov-report xml:coverage.xml --cov-append -vs --disable-warnings \${test_file} 2>&1 | tee ${ut_log_name}echo "##[endgroup]"@@ -104,11 +104,18 @@ function collect_log() {cp coverage.xml ${LOG_DIR}/}+function print_coverage() {+ echo "##[group]overall code coverage..."+ python -m coverage report+ echo "##[endgroup]"+}+function main() {setup_environmentrun_unit_testrun_unit_test_vllmcollect_log+ print_coverageprint_summary}diff --git a/.azure-pipelines/template/ut-template.yml b/.azure-pipelines/template/ut-template.ymlindex 2b6efbe2a..c1a8a9fbe 100644--- a/.azure-pipelines/template/ut-template.yml+++ b/.azure-pipelines/template/ut-template.yml@@ -111,6 +111,15 @@ steps:targetPath: ${{ parameters.uploadPath }}artifact: ${{ parameters.utArtifact }}_coveragepublishLocation: "pipeline"+ displayName: "Publish UT coverage artifact"++ - task: PublishPipelineArtifact@1+ condition: failed()+ inputs:+ targetPath: ${{ parameters.uploadPath }}+ artifact: "${{ parameters.utArtifact }}-$(System.JobAttempt)"+ publishLocation: "pipeline"+ displayName: "Publish UT logs for debugging"- task: Bash@3condition: always()diff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex a01e2d77f..94ff1b199 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -2372,7 +2372,7 @@ def find_layers_from_config(model_dir: str, class_names: list[str] | None = Noneconfig = AutoConfig.from_pretrained(config_dir, trust_remote_code=True)model = AutoModel.from_config(config, trust_remote_code=True)except Exception as e:- logger.warning(f"Failed to load model from {config_dir} for layer detection. Skipping. Error: {e}")+ logger.warning(f"Failed to load model from {config_dir} for layer detection. Skipping. Warning: {e}")continue # skip silentlyfor name, module in model.named_modules():cls_name = type(module).__name__diff --git a/test/test_cuda/requirements.txt b/test/test_cuda/requirements.txtindex cb77b6858..c60f157e9 100644--- a/test/test_cuda/requirements.txt+++ b/test/test_cuda/requirements.txt@@ -8,5 +8,3 @@ pillowtorchvisionnumbasentencepiece-# Remove after new GPTQModel release, https://github.com/intel/auto-round/issues/1891-transformers <= 5.9.0diff --git a/test/test_hpu/test_inference.py b/test/test_hpu/test_inference.pydeleted file mode 100644index 95c680c2d..000000000--- a/test/test_hpu/test_inference.py+++ /dev/null@@ -1,91 +0,0 @@-import shutil--import pytest-import torch-from transformers import AutoModelForCausalLM, AutoTokenizer--from auto_round import AutoRound---def is_hpex_available():- try:- import habana_frameworks.torch.core as htcore # pylint: disable=E0401- except ImportError as e:- return False- return True---# TODO: This test case is temporarily commented out since it not tested for a long time. We need to add it back and change it into pytest format.--# class TestAutoRound:-# @classmethod-# def setup_class(self):-# model_name = "facebook/opt-125m"-# self.model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)-# self.tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)--# @classmethod-# def teardown_class(self):-# shutil.rmtree("./saved", ignore_errors=True)-# shutil.rmtree("runs", ignore_errors=True)--# def test_autogptq_format_hpu_inference(self):-# if not is_hpex_available():-# return-# try:-# import auto_gptq-# except:-# return-# bits, group_size, sym = 4, 128, False-# autoround = AutoRound(-# self.model,-# self.tokenizer,-# bits=bits,-# group_size=group_size,-# sym=sym,-# iters=2,-# seqlen=2,-# dataset=dataloader,-# )-# autoround.quantize()-# quantized_model_path = "./saved"--# autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_gptq")-# model = (-# AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", trust_remote_code=True)-# .to("hpu")-# .to(torch.float32)-# )-# tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)-# text = "There is a girl who likes adventure,"-# inputs = tokenizer(text, return_tensors="pt").to(model.device)-# print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))-# shutil.rmtree("./saved", ignore_errors=True)--# def test_autoround_format_hpu_inference(self):-# if not is_hpex_available():-# return-# bits, group_size, sym = 4, 128, False-# autoround = AutoRound(-# self.model,-# self.tokenizer,-# bits=bits,-# group_size=group_size,-# sym=sym,-# iters=2,-# seqlen=2,-# dataset=dataloader,-# )-# autoround.quantize()-# quantized_model_path = "./saved"--# autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round")--# model = (-# AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto").to("hpu").to(torch.float32)-# )-# tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)-# text = "There is a girl who likes adventure,"-# inputs = tokenizer(text, return_tensors="pt").to(model.device)-# print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))-# shutil.rmtree("./saved", ignore_errors=True)