Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Fix CI coverage & bug grep issue (#1893)

chen, suyuea2fbad794485
diff --git a/.azure-pipelines/scripts/ut/.coverage b/.azure-pipelines/scripts/ut/.coverage
index 5be62fc0d..28b2c2c9a 100644
--- a/.azure-pipelines/scripts/ut/.coverage
+++ b/.azure-pipelines/scripts/ut/.coverage
@@ -3,10 +3,10 @@ branch = True
[paths]
source =
- auto_round/
- /auto-round/auto_round/
+ auto_round
+ */_work/*/s/auto_round
+ */auto-round/auto_round
*/site-packages/auto_round/
- */dist-packages/auto_round/
[report]
include =
diff --git a/.azure-pipelines/scripts/ut/collect_result.py b/.azure-pipelines/scripts/ut/collect_result.py
index 2f0a0c122..7795031bf 100644
--- a/.azure-pipelines/scripts/ut/collect_result.py
+++ b/.azure-pipelines/scripts/ut/collect_result.py
@@ -3,7 +3,6 @@
import argparse
import re
import sys
-from collections import deque
from dataclasses import dataclass
from enum import Enum, auto
from pathlib import Path
@@ -52,15 +51,20 @@ class LogAnalyzer:
)
ERROR_MARKERS = (
- "ERROR",
+ "ERROR:",
"== ERRORS ==",
- " errors",
+ " errors:",
"Exception",
"Traceback",
)
PASS_MARKER = " passed"
+ SKIP_MARKERS = (
+ " deselected",
+ " skipped",
+ )
+
def __init__(self, log_dir: Path, log_pattern: str = "*.log"):
self.log_dir = Path(log_dir)
self.log_pattern = log_pattern
@@ -89,10 +93,10 @@ class LogAnalyzer:
duration=self._extract_duration(content),
)
- def _read_log(self, path: Path, max_lines: int = 100) -> str:
+ def _read_log(self, path: Path) -> str:
try:
with open(path, "r", encoding="utf-8", errors="ignore") as f:
- return "".join(deque(f, maxlen=max_lines))
+ return f.read()
except OSError as e:
print(f"Warning: Cannot read {path} - {e}", file=sys.stderr)
return ""
@@ -112,7 +116,9 @@ class LogAnalyzer:
return TestStatus.FAILED
if self.PASS_MARKER in content:
return TestStatus.PASSED
- return TestStatus.NO_TESTS
+ if any(marker in content for marker in self.SKIP_MARKERS):
+ return TestStatus.NO_TESTS
+ return TestStatus.FAILED
def _extract_duration(self, content: str) -> str:
last_match = None
diff --git a/.azure-pipelines/scripts/ut/run_ut.sh b/.azure-pipelines/scripts/ut/run_ut.sh
index b4dbb261e..0da5c2a7a 100644
--- a/.azure-pipelines/scripts/ut/run_ut.sh
+++ b/.azure-pipelines/scripts/ut/run_ut.sh
@@ -17,14 +17,11 @@ function setup_environment() {
uv pip install pytest-cov pytest-html
uv pip list
- # workaround for ark test, remove auto_round_kernel_xpu
- package_path=$(uv pip show auto-round-lib | grep Location:|cut -d: -f2)
- rm -rf $package_path/auto_round_kernel/auto_round_kernel_xpu*
# install latest gguf for ut test
cd ~ || exit 1
- git clone -b master --quiet --single-branch https://github.com/ggml-org/llama.cpp.git && cd llama.cpp/gguf-py && uv pip install . sentencepiece
-
+ git clone -b master --quiet --single-branch https://github.com/ggml-org/llama.cpp.git && cd llama.cpp/gguf-py && uv pip install .
+
cd /auto-round && uv pip install .
export LD_LIBRARY_PATH=${HOME}/.venv/lib/:$LD_LIBRARY_PATH
@@ -64,7 +61,6 @@ function check_storage_usage() {
function run_unit_test() {
cd /auto-round/test || exit 1
- auto_round_path=$(python -c 'import auto_round; print(auto_round.__path__[0])')
# Split test files into 5 parts
find ./test_cpu -name "test*.py" | grep -Ev "test_llmc|test_inc" | sort > all_tests.txt
@@ -88,7 +84,7 @@ function run_unit_test() {
local ut_log_name=${LOG_DIR}/unittest_${test_basename}.log
numactl --physcpubind="${NUMA_CPUSET:-0-15}" --membind="${NUMA_NODE:-0}" \
- python -m pytest --cov="${auto_round_path}" --cov-report term --html=report.html --self-contained-html \
+ python -m pytest --cov=auto_round --cov-report= --html=report.html --self-contained-html \
--cov-report xml:coverage.xml --cov-append \
-vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
@@ -97,11 +93,10 @@ function run_unit_test() {
function run_inc_unit_test() {
echo "##[group]set up INC UT env..."
- INC_PT_ONLY=1 uv pip install -r /auto-round/test/test_cpu/requirements_inc.txt
+ INC_PT_ONLY=1 uv pip install -r /auto-round/test/test_cpu/requirements_inc.txt --extra-index-url https://download.pytorch.org/whl/cpu
echo "##[endgroup]"
cd /auto-round/test || exit 1
- auto_round_path=$(python -c 'import auto_round; print(auto_round.__path__[0])')
for test_file in $(find ./test_cpu -name "test_inc*.py" | sort); do
echo "##[group]Running ${test_file}..."
@@ -109,7 +104,7 @@ function run_inc_unit_test() {
local ut_log_name=${LOG_DIR}/unittest_${test_basename}.log
numactl --physcpubind="${NUMA_CPUSET:-0-15}" --membind="${NUMA_NODE:-0}" \
- python -m pytest --cov="${auto_round_path}" --cov-report term --html=report.html --self-contained-html \
+ python -m pytest --cov=auto_round --cov-report= --html=report.html --self-contained-html \
--cov-report xml:coverage.xml --cov-append \
-vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
@@ -118,13 +113,12 @@ function run_inc_unit_test() {
function run_llmc_unit_test() {
echo "##[group]set up LLMC UT env..."
- BUILD_TYPE="nightly" uv pip install -r /auto-round/test/test_cpu/requirements_llmc.txt
+ BUILD_TYPE="nightly" uv pip install -r /auto-round/test/test_cpu/requirements_llmc.txt --extra-index-url https://download.pytorch.org/whl/cpu
uv pip uninstall auto-round
cd /auto-round && uv pip install .
echo "##[endgroup]"
cd /auto-round/test || exit 1
- auto_round_path=$(python -c 'import auto_round; print(auto_round.__path__[0])')
for test_file in $(find ./test_cpu -name "test_llmc*.py" | sort); do
echo "##[group]Running ${test_file}..."
@@ -132,7 +126,7 @@ function run_llmc_unit_test() {
local ut_log_name=${LOG_DIR}/unittest_${test_basename}.log
numactl --physcpubind="${NUMA_CPUSET:-0-15}" --membind="${NUMA_NODE:-0}" \
- python -m pytest --cov="${auto_round_path}" --cov-report term --html=report.html --self-contained-html \
+ python -m pytest --cov=auto_round --cov-report= --html=report.html --self-contained-html \
--cov-report xml:coverage.xml --cov-append \
-vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
diff --git a/.azure-pipelines/scripts/ut/run_ut_hpu.sh b/.azure-pipelines/scripts/ut/run_ut_hpu.sh
index e8b36c1d3..4023eddb1 100644
--- a/.azure-pipelines/scripts/ut/run_ut_hpu.sh
+++ b/.azure-pipelines/scripts/ut/run_ut_hpu.sh
@@ -34,7 +34,7 @@ function run_unit_test() {
echo "##[group]Running ${test_file} in HPU lazy mode..."
local ut_log_name="${LOG_DIR}/unittest_lazy_${test_basename}.log"
PT_HPU_LAZY_MODE=1 pytest --cov="${auto_round_path}" \
- --cov-report term --html=report.html --self-contained-html \
+ --cov-report= --html=report.html --self-contained-html \
--cov-report xml:coverage.xml --cov-append -vs --disable-warnings \
${test_file} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
@@ -42,7 +42,7 @@ function run_unit_test() {
echo "##[group]Running ${test_file} in HPU compile mode..."
local ut_log_name="${LOG_DIR}/unittest_compile_${test_basename}.log"
PT_HPU_LAZY_MODE=0 pytest --mode compile --cov="${auto_round_path}" \
- --cov-report term --html=report.html --self-contained-html \
+ --cov-report= --html=report.html --self-contained-html \
--cov-report xml:coverage.xml --cov-append -vs --disable-warnings \
${test_file} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
@@ -74,10 +74,17 @@ function collect_log() {
cp .coverage "${LOG_DIR}/.coverage"
}
+function print_coverage() {
+ echo "##[group]overall code coverage..."
+ python -m coverage report
+ echo "##[endgroup]"
+}
+
function main() {
setup_environment
run_unit_test
collect_log
+ print_coverage
print_summary
}
diff --git a/.azure-pipelines/scripts/ut/run_ut_xpu.sh b/.azure-pipelines/scripts/ut/run_ut_xpu.sh
index 76f405300..5b9e784e3 100644
--- a/.azure-pipelines/scripts/ut/run_ut_xpu.sh
+++ b/.azure-pipelines/scripts/ut/run_ut_xpu.sh
@@ -39,7 +39,7 @@ function run_unit_test() {
echo "##[group]Running ark ${test_file}..."
local ut_log_name="${LOG_DIR}/unittest_ark_${test_basename}.log"
numactl --physcpubind="${NUMA_CPUSET:-0-27}" --membind="${NUMA_NODE:-0}" pytest --cov="${auto_round_path}" \
- --cov-report term --html=report.html --self-contained-html \
+ --cov-report= --html=report.html --self-contained-html \
--cov-report xml:coverage.xml --cov-append -vs --disable-warnings \
${test_file} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
@@ -51,7 +51,7 @@ function run_unit_test() {
echo "##[group]Running xpu ${test_file}..."
local ut_log_name="${LOG_DIR}/unittest_xpu_${test_basename}.log"
numactl --physcpubind="${NUMA_CPUSET:-0-27}" --membind="${NUMA_NODE:-0}" pytest --cov="${auto_round_path}" \
- --cov-report term --html=report.html --self-contained-html \
+ --cov-report= --html=report.html --self-contained-html \
--cov-report xml:coverage.xml --cov-append -vs --disable-warnings \
${test_file} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
@@ -72,7 +72,7 @@ function run_unit_test_vllm() {
echo "##[group]Running xpu vLLM ${test_file}..."
local ut_log_name="${LOG_DIR}/unittest_xpu_${test_basename}.log"
numactl --physcpubind="${NUMA_CPUSET:-0-27}" --membind="${NUMA_NODE:-0}" pytest --cov="${auto_round_path}" \
- --cov-report term --html=report.html --self-contained-html \
+ --cov-report= --html=report.html --self-contained-html \
--cov-report xml:coverage.xml --cov-append -vs --disable-warnings \
${test_file} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
@@ -104,11 +104,18 @@ function collect_log() {
cp coverage.xml ${LOG_DIR}/
}
+function print_coverage() {
+ echo "##[group]overall code coverage..."
+ python -m coverage report
+ echo "##[endgroup]"
+}
+
function main() {
setup_environment
run_unit_test
run_unit_test_vllm
collect_log
+ print_coverage
print_summary
}
diff --git a/.azure-pipelines/template/ut-template.yml b/.azure-pipelines/template/ut-template.yml
index 2b6efbe2a..c1a8a9fbe 100644
--- a/.azure-pipelines/template/ut-template.yml
+++ b/.azure-pipelines/template/ut-template.yml
@@ -111,6 +111,15 @@ steps:
targetPath: ${{ parameters.uploadPath }}
artifact: ${{ parameters.utArtifact }}_coverage
publishLocation: "pipeline"
+ displayName: "Publish UT coverage artifact"
+
+ - task: PublishPipelineArtifact@1
+ condition: failed()
+ inputs:
+ targetPath: ${{ parameters.uploadPath }}
+ artifact: "${{ parameters.utArtifact }}-$(System.JobAttempt)"
+ publishLocation: "pipeline"
+ displayName: "Publish UT logs for debugging"
- task: Bash@3
condition: always()
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.py
index a01e2d77f..94ff1b199 100644
--- a/auto_round/utils/model.py
+++ b/auto_round/utils/model.py
@@ -2372,7 +2372,7 @@ def find_layers_from_config(model_dir: str, class_names: list[str] | None = None
config = AutoConfig.from_pretrained(config_dir, trust_remote_code=True)
model = AutoModel.from_config(config, trust_remote_code=True)
except Exception as e:
- logger.warning(f"Failed to load model from {config_dir} for layer detection. Skipping. Error: {e}")
+ logger.warning(f"Failed to load model from {config_dir} for layer detection. Skipping. Warning: {e}")
continue # skip silently
for name, module in model.named_modules():
cls_name = type(module).__name__
diff --git a/test/test_cuda/requirements.txt b/test/test_cuda/requirements.txt
index cb77b6858..c60f157e9 100644
--- a/test/test_cuda/requirements.txt
+++ b/test/test_cuda/requirements.txt
@@ -8,5 +8,3 @@ pillow
torchvision
numba
sentencepiece
-# Remove after new GPTQModel release, https://github.com/intel/auto-round/issues/1891
-transformers <= 5.9.0
diff --git a/test/test_hpu/test_inference.py b/test/test_hpu/test_inference.py
deleted file mode 100644
index 95c680c2d..000000000
--- a/test/test_hpu/test_inference.py
+++ /dev/null
@@ -1,91 +0,0 @@
-import shutil
-
-import pytest
-import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer
-
-from auto_round import AutoRound
-
-
-def is_hpex_available():
- try:
- import habana_frameworks.torch.core as htcore # pylint: disable=E0401
- except ImportError as e:
- return False
- return True
-
-
-# TODO: This test case is temporarily commented out since it not tested for a long time. We need to add it back and change it into pytest format.
-
-# class TestAutoRound:
-# @classmethod
-# def setup_class(self):
-# model_name = "facebook/opt-125m"
-# self.model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
-# self.tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
-
-# @classmethod
-# def teardown_class(self):
-# shutil.rmtree("./saved", ignore_errors=True)
-# shutil.rmtree("runs", ignore_errors=True)
-
-# def test_autogptq_format_hpu_inference(self):
-# if not is_hpex_available():
-# return
-# try:
-# import auto_gptq
-# except:
-# return
-# bits, group_size, sym = 4, 128, False
-# autoround = AutoRound(
-# self.model,
-# self.tokenizer,
-# bits=bits,
-# group_size=group_size,
-# sym=sym,
-# iters=2,
-# seqlen=2,
-# dataset=dataloader,
-# )
-# autoround.quantize()
-# quantized_model_path = "./saved"
-
-# autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_gptq")
-# model = (
-# AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", trust_remote_code=True)
-# .to("hpu")
-# .to(torch.float32)
-# )
-# tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
-# text = "There is a girl who likes adventure,"
-# inputs = tokenizer(text, return_tensors="pt").to(model.device)
-# print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
-# shutil.rmtree("./saved", ignore_errors=True)
-
-# def test_autoround_format_hpu_inference(self):
-# if not is_hpex_available():
-# return
-# bits, group_size, sym = 4, 128, False
-# autoround = AutoRound(
-# self.model,
-# self.tokenizer,
-# bits=bits,
-# group_size=group_size,
-# sym=sym,
-# iters=2,
-# seqlen=2,
-# dataset=dataloader,
-# )
-# autoround.quantize()
-# quantized_model_path = "./saved"
-
-# autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round")
-
-# model = (
-# AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto").to("hpu").to(torch.float32)
-# )
-# tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
-# text = "There is a girl who likes adventure,"
-# inputs = tokenizer(text, return_tensors="pt").to(model.device)
-# print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
-# shutil.rmtree("./saved", ignore_errors=True)