PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Fix UT coverage (#1927)
Sun, Xuehao6b3118752c41
diff --git a/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh b/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.shindex a5c0fd0b1..98383b95d 100644--- a/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh+++ b/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh@@ -67,7 +67,8 @@ function run_unit_test() {uv pip install 'git+https://github.com/ggml-org/llama.cpp.git#subdirectory=gguf-py'uv pip install -r test/test_cuda/requirements.txtuv pip install -r test/test_cuda/requirements_diffusion.txt- uv pip install -U transformers+ uv pip install -U transformers chardet+ uv pip install kernels==0.12.3 # For sm120: https://github.com/huggingface/transformers/blob/v5.12.1/setup.py#L94uv pip uninstall torch torchvisionuv pip install torch==2.12.0 torchvision torchao --index-url https://download.pytorch.org/whl/cu130uv pip install .@@ -110,11 +111,12 @@ function run_unit_test_llmc() {cd "${BUILD_SOURCESDIRECTORY}" || exit 1rm -rf /root/.venvuv venv --python=3.12 /root/.venv- uv pip install -U pytest-cov pytest-html+ uv pip install -U pytest-covBUILD_TYPE="nightly" uv pip install \-r test/test_cuda/requirements_llmc.txt \--extra-index-url https://download.pytorch.org/whl/cu130 \--index-strategy unsafe-best-match+ uv pip install -U chardetuv pip install .uv pip listecho "##[endgroup]"@@ -139,7 +141,7 @@ function run_unit_test_sglang() {cd "${BUILD_SOURCESDIRECTORY}" || exit 1rm -rf /root/.venvuv venv --python=3.12 /root/.venv- uv pip install -U pytest-cov pytest-html+ uv pip install -U pytest-covuv pip install -r test/test_cuda/requirements_sglang.txt \--prerelease=allow \--extra-index-url https://download.pytorch.org/whl/cu130 \@@ -167,10 +169,11 @@ function run_unit_test_vllm() {cd "${BUILD_SOURCESDIRECTORY}" || exit 1rm -rf /root/.venvuv venv --python=3.12 /root/.venv- uv pip install -U pytest-cov pytest-html+ uv pip install -U pytest-covuv pip install -r test/test_cuda/requirements_vllm.txt \--extra-index-url https://download.pytorch.org/whl/cu130 \--index-strategy unsafe-best-match+ uv pip install -U chardetuv pip install .uv pip listecho "##[endgroup]"diff --git a/.azure-pipelines/scripts/ut/collect_result.py b/.azure-pipelines/scripts/ut/collect_result.pyindex 6c1ac95f3..b3dbb051f 100644--- a/.azure-pipelines/scripts/ut/collect_result.py+++ b/.azure-pipelines/scripts/ut/collect_result.py@@ -59,6 +59,7 @@ class LogAnalyzer:" errors:","Exception","Traceback",+ "Illegal instruction",)PASS_MARKER = " passed"diff --git a/.azure-pipelines/scripts/ut/run_ut.sh b/.azure-pipelines/scripts/ut/run_ut.shindex e29d464f6..76cde6990 100644--- a/.azure-pipelines/scripts/ut/run_ut.sh+++ b/.azure-pipelines/scripts/ut/run_ut.sh@@ -15,7 +15,7 @@ function setup_environment() {export TQDM_MININTERVAL=120export HF_HUB_DISABLE_PROGRESS_BARS=1- uv pip install pytest-cov pytest-html+ uv pip install pytest-covuv pip install -U chardetuv pip list@@ -85,8 +85,7 @@ function run_unit_test() {local ut_log_name=${LOG_DIR}/unittest_${test_basename}.lognumactl --physcpubind="${NUMA_CPUSET:-0-15}" --membind="${NUMA_NODE:-0}" \- python -m pytest --cov=auto_round --cov-report= --html=report.html --self-contained-html \- --cov-report xml:coverage.xml --cov-append \+ pytest --cov=auto_round --cov-report= --cov-append \-vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}echo "##[endgroup]"done@@ -105,8 +104,7 @@ function run_inc_unit_test() {local ut_log_name=${LOG_DIR}/unittest_${test_basename}.lognumactl --physcpubind="${NUMA_CPUSET:-0-15}" --membind="${NUMA_NODE:-0}" \- python -m pytest --cov=auto_round --cov-report= --html=report.html --self-contained-html \- --cov-report xml:coverage.xml --cov-append \+ pytest --cov=auto_round --cov-report= --cov-append \-vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}echo "##[endgroup]"done@@ -127,8 +125,7 @@ function run_llmc_unit_test() {local ut_log_name=${LOG_DIR}/unittest_${test_basename}.lognumactl --physcpubind="${NUMA_CPUSET:-0-15}" --membind="${NUMA_NODE:-0}" \- python -m pytest --cov=auto_round --cov-report= --html=report.html --self-contained-html \- --cov-report xml:coverage.xml --cov-append \+ pytest --cov=auto_round --cov-report= --cov-append \-vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}echo "##[endgroup]"donediff --git a/.azure-pipelines/scripts/ut/run_ut_cuda.sh b/.azure-pipelines/scripts/ut/run_ut_cuda.shindex 7391a537e..92f715c7d 100644--- a/.azure-pipelines/scripts/ut/run_ut_cuda.sh+++ b/.azure-pipelines/scripts/ut/run_ut_cuda.sh@@ -31,7 +31,7 @@ function create_conda_env() {export PATH=/usr/local/cuda/bin${PATH:+:${PATH}}export LD_LIBRARY_PATH=$(python -c "import site; print(site.getsitepackages()[0])")/nvidia/nvjitlink/lib:$LD_LIBRARY_PATHfi- uv pip install pytest-cov pytest-html cmake requests+ uv pip install pytest-cov cmake requests}function print_test_results_table() {@@ -92,7 +92,7 @@ function run_unit_test() {rm -rf .coverage* *.xml *.htmluv pip install torch==2.12.0 torchvision torchao --index-url https://download.pytorch.org/whl/cu126- uv pip install llama-cpp-python --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu125+ CMAKE_ARGS="-DGGML_CUDA=on -DLLAVA_BUILD=off" uv pip install llama-cpp-pythonuv pip install 'git+https://github.com/ggml-org/llama.cpp.git#subdirectory=gguf-py'uv pip install -r test_cuda/requirements.txtuv pip install -r test_cuda/requirements_diffusion.txt@@ -103,7 +103,6 @@ function run_unit_test() {pip list > ${LOG_DIR}/ut_pip_list.txtexport COVERAGE_RCFILE=${REPO_PATH}/.azure-pipelines/scripts/ut/.coverage- local auto_round_path=$(python -c 'import auto_round; print(auto_round.__path__[0])')# run unit tests individually with separate logsfor test_file in $(find ./test_cuda -type f -name "test_*.py" | grep -Ev "vlms|llmc|sglang|vllm|multiple_card" | sort); do@@ -111,13 +110,9 @@ function run_unit_test() {local ut_log_name=${LOG_DIR}/unittest_cuda_${test_basename}.logecho "Running ${test_file}..."- python -m pytest --cov="${auto_round_path}" --cov-report term --html=report.html --self-contained-html --cov-report xml:coverage.xml --cov-append -vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}+ pytest --cov=auto_round --cov-report= --cov-append -vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}done-- if [ -f "report.html" ] && [ -f "coverage.xml" ]; then- mv report.html ${LOG_DIR}/- mv coverage.xml ${LOG_DIR}/- fi+ [ -f .coverage ] && cp .coverage ${LOG_DIR}/.coverage.unit# Print test results table and check for failuresif ! print_test_results_table "unittest_cuda_test_*.log" "CUDA Unit Tests"; then@@ -143,7 +138,6 @@ function run_unit_test_vlm() {pip list > ${LOG_DIR}/vlm_ut_pip_list.txtexport COVERAGE_RCFILE=${REPO_PATH}/.azure-pipelines/scripts/ut/.coverage- local auto_round_path=$(python -c 'import auto_round; print(auto_round.__path__[0])')# run VLM unit tests individually with separate logsfor test_file in $(find ./test_cuda -name "test*vlms.py"); do@@ -151,13 +145,9 @@ function run_unit_test_vlm() {local ut_log_name=${LOG_DIR}/unittest_cuda_vlm_${test_basename}.logecho "Running ${test_file}..."- python -m pytest --cov="${auto_round_path}" --cov-report term --html=report_vlms.html --self-contained-html --cov-report xml:coverage_vlms.xml --cov-append -vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}+ pytest --cov=auto_round --cov-report= --cov-append -vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}done-- if [ -f "report_vlms.html" ] && [ -f "coverage_vlms.xml" ]; then- mv report_vlms.html ${LOG_DIR}/- mv coverage_vlms.xml ${LOG_DIR}/- fi+ [ -f .coverage ] && cp .coverage ${LOG_DIR}/.coverage.vlm# Print test results table and check for failuresif ! print_test_results_table "unittest_cuda_vlm_test*.log" "CUDA VLM Tests"; then@@ -177,7 +167,6 @@ function run_unit_test_llmc() {pip list > ${LOG_DIR}/llmc_ut_pip_list.txtexport COVERAGE_RCFILE=${REPO_PATH}/.azure-pipelines/scripts/ut/.coverage- local auto_round_path=$(python -c 'import auto_round; print(auto_round.__path__[0])')# run unit tests individually with separate logsfor test_file in $(find ./test_cuda -name "test_llmc*.py" | sort); do@@ -185,13 +174,10 @@ function run_unit_test_llmc() {local ut_log_name=${LOG_DIR}/unittest_cuda_llmc_${test_basename}.logecho "Running ${test_file}..."- python -m pytest --cov="${auto_round_path}" --cov-report term --html=report_llmc.html --self-contained-html --cov-report xml:coverage_llmc.xml --cov-append -vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}+ pytest --cov=auto_round --cov-report= --cov-append -vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}done+ [ -f .coverage ] && cp .coverage ${LOG_DIR}/.coverage.llmc- if [ -f "report_llmc.html" ] && [ -f "coverage_llmc.xml" ]; then- mv report_llmc.html ${LOG_DIR}/- mv coverage_llmc.xml ${LOG_DIR}/- fi# Print test results table and check for failuresif ! print_test_results_table "unittest_cuda_llmc_test_*.log" "CUDA LLMC Tests"; thenecho "Some CUDA LLMC tests failed. Please check the individual log files for details."@@ -209,7 +195,6 @@ function run_unit_test_sglang() {pip list > ${LOG_DIR}/sglang_ut_pip_list.txtexport COVERAGE_RCFILE=${REPO_PATH}/.azure-pipelines/scripts/ut/.coverage- local auto_round_path=$(python -c 'import auto_round; print(auto_round.__path__[0])')# run unit tests individually with separate logsfor test_file in $(find ./test_cuda -name "test_sglang*.py" | sort); do@@ -217,13 +202,10 @@ function run_unit_test_sglang() {local ut_log_name=${LOG_DIR}/unittest_cuda_sglang_${test_basename}.logecho "Running ${test_file}..."- python -m pytest --cov="${auto_round_path}" --cov-report term --html=report_sglang.html --self-contained-html --cov-report xml:coverage_sglang.xml --cov-append -vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}+ pytest --cov=auto_round --cov-report= --cov-append -vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}done+ [ -f .coverage ] && cp .coverage ${LOG_DIR}/.coverage.sglang- if [ -f "report_sglang.html" ] && [ -f "coverage_sglang.xml" ]; then- mv report_sglang.html ${LOG_DIR}/- mv coverage_sglang.xml ${LOG_DIR}/- fi# Print test results table and check for failuresif ! print_test_results_table "unittest_cuda_sglang_test*.log" "CUDA SGLang Unit Tests"; thenecho "Some CUDA SGLang unit tests failed. Please check the individual log files for details."@@ -247,7 +229,6 @@ function run_unit_test_vllm() {pip list > ${LOG_DIR}/vllm_ut_pip_list.txtexport COVERAGE_RCFILE=${REPO_PATH}/.azure-pipelines/scripts/ut/.coverage- local auto_round_path=$(python -c 'import auto_round; print(auto_round.__path__[0])')# run unit tests individually with separate logsfor test_file in $(find ./test_cuda -name "test_vllm*.py" | sort); do@@ -255,25 +236,41 @@ function run_unit_test_vllm() {local ut_log_name=${LOG_DIR}/unittest_cuda_vllm_${test_basename}.logecho "Running ${test_file}..."- python -m pytest --cov="${auto_round_path}" --cov-report term --html=report_vllm.html --self-contained-html --cov-report xml:coverage_vllm.xml --cov-append -vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}+ pytest --cov=auto_round --cov-report= --cov-append -vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}done+ [ -f .coverage ] && cp .coverage ${LOG_DIR}/.coverage.vllm- if [ -f "report_vllm.html" ] && [ -f "coverage_vllm.xml" ]; then- mv report_vllm.html ${LOG_DIR}/- mv coverage_vllm.xml ${LOG_DIR}/- fi# Print test results table and check for failuresif ! print_test_results_table "unittest_cuda_vllm_test*.log" "CUDA VLLM Unit Tests"; thenecho "Some CUDA VLLM unit tests failed. Please check the individual log files for details."fi}+function merge_coverage() {+ echo "-----[VAL INFO] merging coverage data -----"+ cd ${REPO_PATH}/test++ local coverage_files=$(find ${LOG_DIR} -maxdepth 1 -name ".coverage.*" 2>/dev/null)+ if [ -z "${coverage_files}" ]; then+ echo "No coverage data files found in ${LOG_DIR}, skipping merge."+ return+ fi++ export COVERAGE_RCFILE=${REPO_PATH}/.azure-pipelines/scripts/ut/.coverage+ coverage combine ${coverage_files}+ coverage xml -o ${LOG_DIR}/coverage_merged.xml+ coverage html -d ${LOG_DIR}/htmlcov+ coverage report | tee ${LOG_DIR}/coverage_summary.log+ echo "Merged coverage report saved to ${LOG_DIR}/coverage_merged.xml and ${LOG_DIR}/htmlcov/index.html"+}+function main() {run_unit_test_vllmrun_unit_test_vlmrun_unit_test_llmcrun_unit_test_sglangrun_unit_test+ merge_coveragecat ${SUMMARY_LOG}}diff --git a/.azure-pipelines/scripts/ut/run_ut_hpu.sh b/.azure-pipelines/scripts/ut/run_ut_hpu.shindex 4023eddb1..46c731845 100644--- a/.azure-pipelines/scripts/ut/run_ut_hpu.sh+++ b/.azure-pipelines/scripts/ut/run_ut_hpu.sh@@ -8,7 +8,7 @@ function setup_environment() {export TZ='Asia/Shanghai'export TQDM_MININTERVAL=60export HF_HUB_DISABLE_PROGRESS_BARS=1- pip install pytest-cov pytest-html+ pip install pytest-covpip listecho "##[endgroup]"@@ -34,16 +34,14 @@ function run_unit_test() {echo "##[group]Running ${test_file} in HPU lazy mode..."local ut_log_name="${LOG_DIR}/unittest_lazy_${test_basename}.log"PT_HPU_LAZY_MODE=1 pytest --cov="${auto_round_path}" \- --cov-report= --html=report.html --self-contained-html \- --cov-report xml:coverage.xml --cov-append -vs --disable-warnings \+ --cov-report= --cov-append -vs --disable-warnings \${test_file} 2>&1 | tee ${ut_log_name}echo "##[endgroup]"echo "##[group]Running ${test_file} in HPU compile mode..."local ut_log_name="${LOG_DIR}/unittest_compile_${test_basename}.log"PT_HPU_LAZY_MODE=0 pytest --mode compile --cov="${auto_round_path}" \- --cov-report= --html=report.html --self-contained-html \- --cov-report xml:coverage.xml --cov-append -vs --disable-warnings \+ --cov-report= --cov-append -vs --disable-warnings \${test_file} 2>&1 | tee ${ut_log_name}echo "##[endgroup]"done@@ -69,9 +67,9 @@ function print_summary() {function collect_log() {python /auto-round/.azure-pipelines/scripts/ut/collect_result.py \--test-type "Unit Tests" --log-pattern "unittest_*.log" --log-dir ${LOG_DIR} --summary-log ${SUMMARY_LOG}- cp report.html ${LOG_DIR}/- cp coverage.xml ${LOG_DIR}/cp .coverage "${LOG_DIR}/.coverage"+ python -m coverage xml -o "${LOG_DIR}/coverage.xml"+ python -m coverage html -d "${LOG_DIR}/htmlcov"}function print_coverage() {diff --git a/.azure-pipelines/scripts/ut/run_ut_xpu.sh b/.azure-pipelines/scripts/ut/run_ut_xpu.shindex 5b9e784e3..30b06bdff 100644--- a/.azure-pipelines/scripts/ut/run_ut_xpu.sh+++ b/.azure-pipelines/scripts/ut/run_ut_xpu.sh@@ -5,7 +5,7 @@ source /auto-round/.azure-pipelines/scripts/change_color.shfunction setup_environment() {echo "##[group]set up UT env..."- uv pip install pytest-cov pytest-html+ uv pip install pytest-covuv pip listecho "##[endgroup]"@@ -38,9 +38,8 @@ function run_unit_test() {echo "##[group]Running ark ${test_file}..."local ut_log_name="${LOG_DIR}/unittest_ark_${test_basename}.log"- numactl --physcpubind="${NUMA_CPUSET:-0-27}" --membind="${NUMA_NODE:-0}" pytest --cov="${auto_round_path}" \- --cov-report= --html=report.html --self-contained-html \- --cov-report xml:coverage.xml --cov-append -vs --disable-warnings \+ numactl --physcpubind="${NUMA_CPUSET:-0-27}" --membind="${NUMA_NODE:-0}" \+ pytest --cov="${auto_round_path}" --cov-report= --cov-append -vs --disable-warnings \${test_file} 2>&1 | tee ${ut_log_name}echo "##[endgroup]"done@@ -50,9 +49,8 @@ function run_unit_test() {echo "##[group]Running xpu ${test_file}..."local ut_log_name="${LOG_DIR}/unittest_xpu_${test_basename}.log"- numactl --physcpubind="${NUMA_CPUSET:-0-27}" --membind="${NUMA_NODE:-0}" pytest --cov="${auto_round_path}" \- --cov-report= --html=report.html --self-contained-html \- --cov-report xml:coverage.xml --cov-append -vs --disable-warnings \+ numactl --physcpubind="${NUMA_CPUSET:-0-27}" --membind="${NUMA_NODE:-0}" \+ pytest --cov="${auto_round_path}" --cov-report= --cov-append -vs --disable-warnings \${test_file} 2>&1 | tee ${ut_log_name}echo "##[endgroup]"done@@ -71,9 +69,8 @@ function run_unit_test_vllm() {echo "##[group]Running xpu vLLM ${test_file}..."local ut_log_name="${LOG_DIR}/unittest_xpu_${test_basename}.log"- numactl --physcpubind="${NUMA_CPUSET:-0-27}" --membind="${NUMA_NODE:-0}" pytest --cov="${auto_round_path}" \- --cov-report= --html=report.html --self-contained-html \- --cov-report xml:coverage.xml --cov-append -vs --disable-warnings \+ numactl --physcpubind="${NUMA_CPUSET:-0-27}" --membind="${NUMA_NODE:-0}" \+ pytest --cov="${auto_round_path}" --cov-report= --cov-append -vs --disable-warnings \${test_file} 2>&1 | tee ${ut_log_name}echo "##[endgroup]"done@@ -100,8 +97,8 @@ function collect_log() {python /auto-round/.azure-pipelines/scripts/ut/collect_result.py \--test-type "Unit Tests" --log-pattern "unittest_*.log" --log-dir ${LOG_DIR} --summary-log ${SUMMARY_LOG}cp .coverage "${LOG_DIR}/.coverage"- cp report.html ${LOG_DIR}/- cp coverage.xml ${LOG_DIR}/+ python -m coverage xml -o "${LOG_DIR}/coverage.xml"+ python -m coverage html -d "${LOG_DIR}/htmlcov"}function print_coverage() {diff --git a/.azure-pipelines/unit-test-cuda.yml b/.azure-pipelines/unit-test-cuda.ymlindex be97904ff..ba6baca35 100644--- a/.azure-pipelines/unit-test-cuda.yml+++ b/.azure-pipelines/unit-test-cuda.yml@@ -138,7 +138,7 @@ stages:- script: |echo "##[group]Installing Python dependencies..."- uv pip install -U "pytest-cov" "pytest-html" "requests" "huggingface_hub"+ uv pip install -U "pytest-cov" "requests" "huggingface_hub"hf auth login --token "$(HF_TOKEN)"echo "##[endgroup]"displayName: "Install Python Dependencies"diff --git a/README.md b/README.mdindex 30afa4a05..90524ca2f 100644--- a/README.md+++ b/README.md@@ -10,7 +10,7 @@<h3> Advanced Quantization Algorithm for LLMs</h3>[](https://github.com/intel/auto-round)-[](https://github.com/intel/auto-round/releases)+[](https://github.com/intel/auto-round/releases)[](https://pypi.org/project/auto-round-nightly)[](https://github.com/intel/auto-round/blob/main/LICENSE)<a href="https://huggingface.co/Intel">diff --git a/README_CN.md b/README_CN.mdindex 7641b9e48..d87e77278 100644--- a/README_CN.md+++ b/README_CN.md@@ -10,7 +10,7 @@<h3> 面向 LLM 的先进量化算法</h3>[](https://github.com/intel/auto-round)-[](https://github.com/intel/auto-round/releases)+[](https://github.com/intel/auto-round/releases)[](https://pypi.org/project/auto-round-nightly)[](https://github.com/intel/auto-round/blob/main/LICENSE)<a href="https://huggingface.co/Intel">diff --git a/test/test_cuda/integrations/test_vllm.py b/test/test_cuda/integrations/test_vllm.pyindex 63636e395..bc1b65ac2 100644--- a/test/test_cuda/integrations/test_vllm.py+++ b/test/test_cuda/integrations/test_vllm.py@@ -83,6 +83,7 @@ def test_auto_round(model):prompt = output.promptgenerated_text = output.outputs[0].textif "France" in prompt:+ print(generated_text)assert "Paris" in generated_text@@ -202,6 +203,7 @@ def test_vllm_awq_w8a8_llmc_inference(tiny_opt_model_path, tmp_path):)outputs = llm.generate(["Hello, my name is"], sampling_params)generated_text = outputs[0].outputs[0].text+ print(generated_text)assert len(generated_text.strip()) > 0 and "!!!" not in generated_text, "vLLM produced empty/meaningless output"