Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Optimize CPU unit test workflow (#881)

Sun, Xuehao6e567b72ca0c
diff --git a/.azure-pipelines/docker/Dockerfile.devel b/.azure-pipelines/docker/Dockerfile.devel
index 818262ae7..397dfeeab 100644
--- a/.azure-pipelines/docker/Dockerfile.devel
+++ b/.azure-pipelines/docker/Dockerfile.devel
@@ -45,6 +45,7 @@ RUN groupadd -g ${GROUP_ID} hostgroup && \
USER hostuser
+ENV PATH="/home/hostuser/.local/bin:${PATH}"
RUN python -m pip install --no-cache-dir --upgrade pip
RUN python -m pip install --no-cache-dir setuptools
diff --git a/.azure-pipelines/docker/DockerfileCodeScan.devel b/.azure-pipelines/docker/DockerfileCodeScan.devel
index 7eba9068d..7944cea1d 100644
--- a/.azure-pipelines/docker/DockerfileCodeScan.devel
+++ b/.azure-pipelines/docker/DockerfileCodeScan.devel
@@ -40,6 +40,8 @@ RUN groupadd -g ${GROUP_ID} hostgroup && \
USER hostuser
+ENV PATH="/home/hostuser/.local/bin:${PATH}"
+RUN python -m pip install --no-cache-dir --upgrade pip
RUN python -m pip install --no-cache-dir pylint==2.12.1\
bandit
diff --git a/.azure-pipelines/scripts/ut/.coverage b/.azure-pipelines/scripts/ut/.coverage
index 659652406..7ac7a0fb2 100644
--- a/.azure-pipelines/scripts/ut/.coverage
+++ b/.azure-pipelines/scripts/ut/.coverage
@@ -1,6 +1,12 @@
[run]
branch = True
+[paths]
+source =
+ auto_round/
+ */site-packages/auto_round/
+ */dist-packages/auto_round/
+
[report]
include =
*/auto_round/**
diff --git a/.azure-pipelines/scripts/ut/collect_log.sh b/.azure-pipelines/scripts/ut/collect_log.sh
index c1a27aebf..54d5cb5b7 100644
--- a/.azure-pipelines/scripts/ut/collect_log.sh
+++ b/.azure-pipelines/scripts/ut/collect_log.sh
@@ -1,133 +1,19 @@
-source /auto-round/.azure-pipelines/scripts/change_color.sh
-
+#!/bin/bash
set -e
pip install coverage
export COVERAGE_RCFILE=/auto-round/.azure-pipelines/scripts/ut/.coverage
coverage_log="/auto-round/log_dir/coverage_log"
-coverage_log_base="/auto-round/log_dir/coverage_log_base"
-coverage_compare="/auto-round/log_dir/coverage_compare.html"
cd /auto-round/log_dir
-$BOLD_YELLOW && echo "collect coverage for PR branch" && $RESET
-cp ut_coverage/.coverage /auto-round/
+echo "collect coverage for PR branch"
mkdir -p coverage_PR
+cp ut-*/.coverage.* ./coverage_PR/
+cd coverage_PR
+coverage combine --keep --rcfile=${COVERAGE_RCFILE}
+
+cp .coverage /auto-round/
cd /auto-round
coverage report -m --rcfile=${COVERAGE_RCFILE} | tee ${coverage_log}
coverage html -d log_dir/coverage_PR/htmlcov --rcfile=${COVERAGE_RCFILE}
coverage xml -o log_dir/coverage_PR/coverage.xml --rcfile=${COVERAGE_RCFILE}
ls -l log_dir/coverage_PR/htmlcov
-
-
-$BOLD_YELLOW && echo "collect coverage for baseline" && $RESET
-cd /auto-round
-cp -r /auto-round/.azure-pipelines .azure-pipelines-pr
-git config --global --add safe.directory /auto-round
-git fetch
-git checkout main
-rm -rf build dist *egg-info
-echo y | pip uninstall auto_round
-pip install -r requirements.txt
-pip install -vvv --no-build-isolation -e .[cpu]
-
-coverage erase
-cd /auto-round/log_dir
-mkdir -p coverage_base
-rm -rf /auto-round/.coverage || true
-cp ut_baseline_coverage/.coverage /auto-round
-
-cd /auto-round
-coverage report -m --rcfile=${COVERAGE_RCFILE} | tee ${coverage_log_base}
-coverage html -d log_dir/coverage_base/htmlcov --rcfile=${COVERAGE_RCFILE}
-coverage xml -o log_dir/coverage_base/coverage.xml --rcfile=${COVERAGE_RCFILE}
-ls -l log_dir/coverage_base/htmlcov
-
-get_coverage_data() {
- # Input argument
- local coverage_xml="$1"
-
- # Get coverage data
- local coverage_data=$(python3 -c "import xml.etree.ElementTree as ET; root = ET.parse('$coverage_xml').getroot(); print(ET.tostring(root).decode())")
- if [[ -z "$coverage_data" ]]; then
- echo "Failed to get coverage data from $coverage_xml."
- exit 1
- fi
-
- # Get lines coverage
- local lines_covered=$(echo "$coverage_data" | grep -o 'lines-covered="[0-9]*"' | cut -d '"' -f 2)
- local lines_valid=$(echo "$coverage_data" | grep -o 'lines-valid="[0-9]*"' | cut -d '"' -f 2)
- if [ $lines_valid == 0 ]; then
- local lines_coverage=0
- else
- local lines_coverage=$(awk "BEGIN {printf \"%.3f\", 100 * $lines_covered / $lines_valid}")
- fi
-
- # Get branches coverage
- local branches_covered=$(echo "$coverage_data" | grep -o 'branches-covered="[0-9]*"' | cut -d '"' -f 2)
- local branches_valid=$(echo "$coverage_data" | grep -o 'branches-valid="[0-9]*"' | cut -d '"' -f 2)
- if [ $branches_valid == 0 ]; then
- local branches_coverage=0
- else
- local branches_coverage=$(awk "BEGIN {printf \"%.3f\", 100 * $branches_covered/$branches_valid}")
- fi
-
- # Return values
- echo "$lines_covered $lines_valid $lines_coverage $branches_covered $branches_valid $branches_coverage"
-}
-
-$BOLD_YELLOW && echo "compare coverage" && $RESET
-
-coverage_PR_xml="log_dir/coverage_PR/coverage.xml"
-coverage_PR_data=$(get_coverage_data $coverage_PR_xml)
-read lines_PR_covered lines_PR_valid coverage_PR_lines_rate branches_PR_covered branches_PR_valid coverage_PR_branches_rate <<<"$coverage_PR_data"
-
-coverage_base_xml="log_dir/coverage_base/coverage.xml"
-coverage_base_data=$(get_coverage_data $coverage_base_xml)
-read lines_base_covered lines_base_valid coverage_base_lines_rate branches_base_covered branches_base_valid coverage_base_branches_rate <<<"$coverage_base_data"
-
-$BOLD_BLUE && echo "PR lines coverage: $lines_PR_covered/$lines_PR_valid ($coverage_PR_lines_rate%)" && $RESET
-$BOLD_BLUE && echo "PR branches coverage: $branches_PR_covered/$branches_PR_valid ($coverage_PR_branches_rate%)" && $RESET
-$BOLD_BLUE && echo "BASE lines coverage: $lines_base_covered/$lines_base_valid ($coverage_base_lines_rate%)" && $RESET
-$BOLD_BLUE && echo "BASE branches coverage: $branches_base_covered/$branches_base_valid ($coverage_base_branches_rate%)" && $RESET
-
-$BOLD_YELLOW && echo "clear upload path" && $RESET
-rm -fr log_dir/coverage_PR/.coverage*
-rm -fr log_dir/coverage_base/.coverage*
-rm -fr log_dir/ut-coverage-*
-
-# Declare an array to hold failed items
-declare -a fail_items=()
-
-if (( $(bc -l <<< "${coverage_PR_lines_rate}+0.05 < ${coverage_base_lines_rate}") )); then
- fail_items+=("lines")
-fi
-if (( $(bc -l <<< "${coverage_PR_branches_rate}+0.05 < ${coverage_base_branches_rate}") )); then
- fail_items+=("branches")
-fi
-
-if [[ ${#fail_items[@]} -ne 0 ]]; then
- fail_items_str=$(
- IFS=', '
- echo "${fail_items[*]}"
- )
- for item in "${fail_items[@]}"; do
- case "$item" in
- lines)
- decrease=$(echo $(printf "%.3f" $(echo "$coverage_PR_lines_rate - $coverage_base_lines_rate" | bc -l)))
- ;;
- branches)
- decrease=$(echo $(printf "%.3f" $(echo "$coverage_PR_branches_rate - $coverage_base_branches_rate" | bc -l)))
- ;;
- *)
- echo "Unknown item: $item"
- continue
- ;;
- esac
- $BOLD_RED && echo "Unit Test failed with ${item} coverage decrease ${decrease}%" && $RESET
- done
- $BOLD_RED && echo "compare coverage to give detail info" && $RESET
- bash /auto-round/.azure-pipelines-pr/scripts/ut/compare_coverage.sh ${coverage_compare} ${coverage_log} ${coverage_log_base} "FAILED" ${coverage_PR_lines_rate} ${coverage_base_lines_rate} ${coverage_PR_branches_rate} ${coverage_base_branches_rate}
-else
- $BOLD_GREEN && echo "Unit Test success with coverage lines: ${coverage_PR_lines_rate}%, branches: ${coverage_PR_branches_rate}%" && $RESET
- $BOLD_GREEN && echo "compare coverage to give detail info" && $RESET
- bash /auto-round/.azure-pipelines-pr/scripts/ut/compare_coverage.sh ${coverage_compare} ${coverage_log} ${coverage_log_base} "SUCCESS" ${coverage_PR_lines_rate} ${coverage_base_lines_rate} ${coverage_PR_branches_rate} ${coverage_base_branches_rate}
-fi
diff --git a/.azure-pipelines/scripts/ut/compare_coverage.sh b/.azure-pipelines/scripts/ut/compare_coverage.sh
deleted file mode 100644
index b0d6442ee..000000000
--- a/.azure-pipelines/scripts/ut/compare_coverage.sh
+++ /dev/null
@@ -1,225 +0,0 @@
-output_file=$1
-coverage_pr_log=$2
-coverage_base_log=$3
-coverage_status=$4
-coverage_PR_lines_rate=$5
-coverage_base_lines_rate=$6
-coverage_PR_branches_rate=$7
-coverage_base_branches_rate=$8
-module_name="auto-round"
-[[ ! -f $coverage_pr_log ]] && exit 1
-[[ ! -f $coverage_base_log ]] && exit 1
-file_name="./coverage_compare"
-sed -i "s|\/usr.*${module_name}\/||g" $coverage_pr_log
-sed -i "s|\/usr.*${module_name}\/||g" $coverage_base_log
-diff $coverage_pr_log $coverage_base_log >diff_file
-[[ $? == 0 ]] && exit 0
-grep -Po "[<,>,\d].*" diff_file | awk '{print $1 "\t" $2 "\t" $3 "\t" $4 "\t" $5 "\t" $6 "\t" $7}' | sed "/Name/d" | sed "/TOTAL/d" | sed "/---/d" >$file_name
-[[ ! -s $file_name ]] && exit 0
-[[ -f $output_file ]] && rm -f $output_file
-touch $output_file
-
-function generate_html_head {
-
- cat >${output_file} <<eof
-
-<!DOCTYPE html>
-<html lang="en">
-<head>
- <meta charset="UTF-8">
- <meta http-equiv="X-UA-Compatible" content="IE=edge">
- <meta name="viewport" content="width=device-width, initial-scale=1.0">
- <title>UT coverage</title>
- <style type="text/css">
- body {
- margin: 0;
- padding: 0;
- background: white no-repeat left top;
- }
-
- .main {
- margin: 20px auto 10px auto;
- background: white;
- border-radius: 8px;
- -moz-border-radius: 8px;
- -webkit-border-radius: 8px;
- padding: 0 30px 30px 30px;
- border: 1px solid #adaa9f;
- box-shadow: 0 2px 2px #9c9c9c;
- -moz-box-shadow: 0 2px 2px #9c9c9c;
- -webkit-box-shadow: 0 2px 2px #9c9c9c;
- }
-
- .features-table {
- width: 100%;
- margin: 0 auto;
- border-collapse: separate;
- border-spacing: 0;
- text-shadow: 0 1px 0 #fff;
- color: #2a2a2a;
- background: #fafafa;
- background-image: -moz-linear-gradient(top, #fff, #eaeaea, #fff);
- /* Firefox 3.6 */
- background-image: -webkit-gradient(linear, center bottom, center top, from(#fff), color-stop(0.5, #eaeaea), to(#fff));
- font-family: Verdana, Arial, Helvetica
- }
-
- .features-table th,
- td {
- text-align: center;
- height: 25px;
- line-height: 25px;
- padding: 0 8px;
- border: 1px solid #cdcdcd;
- box-shadow: 0 1px 0 white;
- -moz-box-shadow: 0 1px 0 white;
- -webkit-box-shadow: 0 1px 0 white;
- white-space: nowrap;
- }
- </style>
-</head>
-
-eof
-}
-
-function extract_diff_data() {
- local file_name=$1 diff_file=$2 reg=$3
- local file=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po "${reg}.*" | sed "s/${reg}[ \t]*//g" | awk '{print $1}')
- local stmts=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po "${reg}.*" | sed "s/${reg}[ \t]*//g" | awk '{print $2}')
- local miss=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po "${reg}.*" | sed "s/${reg}[ \t]*//g" | awk '{print $3}')
- local cover=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po "${reg}.*" | sed "s/${reg}[ \t]*//g" | awk '{print $6}')
- local branch=$(cat $file_name | grep "${diff_file}" | grep -v ".*/${diff_file}" | grep -Po "${reg}.*" | sed "s/${reg}[ \t]*//g" | awk '{print $4}')
-
- echo "$file $stmts $miss $cover $branch"
-}
-
-function write_compare_details() {
- local file=$1 stmts1=$2 miss1=$3 branch1=$4 cover1=$5 stmts2=$6 miss2=$7 branch2=$8 cover2=$9
- echo """
- <tr>
- <td>PR | BASE</td>
- <td style=\"text-align:left\">${file}</td>
- <td style=\"text-align:left\">${stmts1} | ${stmts2}</td>
- <td style=\"text-align:left\">${miss1} | ${miss2}</td>
- <td style=\"text-align:left\">${branch1} | ${branch2}</td>
- <td style=\"text-align:left\">${cover1} | ${cover2}</td>
- </tr>
- """ >>${output_file}
-}
-
-function get_color() {
- local decrease=$1
- if (($(echo "$decrease < 0" | bc -l))); then
- local color="#FFD2D2"
- else
- local color="#90EE90"
- fi
- echo "$color"
-}
-
-function generate_coverage_summary() {
- # generate table head
- local Lines_cover_decrease=$(echo $(printf "%.3f" $(echo "$coverage_PR_lines_rate - $coverage_base_lines_rate" | bc -l)))
- local Branches_cover_decrease=$(echo $(printf "%.3f" $(echo "$coverage_PR_branches_rate - $coverage_base_branches_rate" | bc -l)))
-
- read lines_coverage_color <<<"$(get_color ${Lines_cover_decrease})"
- read branches_coverage_color <<<"$(get_color ${Branches_cover_decrease})"
-
- echo """
-<body>
- <div class="main">
- <h1 align="center">Coverage Summary : ${coverage_status}</h1>
- <table class=\"features-table\" style=\"width: 60%;margin-left:auto;margin-right:auto;empty-cells: hide\">
- <tr>
- <th></th>
- <th>Base coverage</th>
- <th>PR coverage</th>
- <th>Diff</th>
- </tr>
- <tr>
- <td> Lines </td>
- <td> ${coverage_base_lines_rate}% </td>
- <td> ${coverage_PR_lines_rate}% </td>
- <td style=\"background-color:${lines_coverage_color}\"> ${Lines_cover_decrease}% </td>
- </tr>
- <tr>
- <td> Branches </td>
- <td> ${coverage_base_branches_rate}% </td>
- <td> ${coverage_PR_branches_rate}% </td>
- <td style=\"background-color:${branches_coverage_color}\"> ${Branches_cover_decrease}% </td>
- </tr>
- </table>
- </div>
- """ >>${output_file}
-}
-
-function generate_coverage_details() {
- echo """
- <div class="main">
- <h2 align="center">Coverage Detail</h2>
- <table class=\"features-table\" style=\"width: 60%;margin-left:auto;margin-right:auto;empty-cells: hide\">
- <tr>
- <th>Commit</th>
- <th>FileName</th>
- <th>Stmts</th>
- <th>Miss</th>
- <th>Branch</th>
- <th>Cover</th>
- </tr>
- """ >>${output_file}
- # generate compare detail
- cat ${file_name} | while read line; do
- if [[ $(echo $line | grep "[0-9]a[0-9]") ]] && [[ $(grep -A 1 "$line" ${file_name} | grep ">") ]]; then
- diff_lines=$(sed -n "/${line}/,/^[0-9]/p" ${file_name} | grep ">")
- diff_file_name=$(sed -n "/${line}/,/^[0-9]/p" ${file_name} | grep -Po ">.*[a-z,A-Z].*.py" | sed "s|>||g")
- for diff_file in ${diff_file_name}; do
- diff_file=$(echo "${diff_file}" | sed 's/[ \t]*//g')
- diff_coverage_data=$(extract_diff_data ${file_name} ${diff_file} ">")
- read file stmts miss cover branch <<<"$diff_coverage_data"
- write_compare_details $file "NA" "NA" "NA" "NA" $stmts $miss $branch $cover
- done
- elif [[ $(echo $line | grep "[0-9]c[0-9]") ]] && [[ $(cat ${file_name} | grep -A 1 "$line" | grep "<") ]]; then
- diff_lines=$(sed -n "/${line}/,/^[0-9]/p" ${file_name} | grep "<")
- diff_file_name=$(sed -n "/${line}/,/^[0-9]/p" ${file_name} | grep -Po "<.*[a-z,A-Z].*.py" | sed "s|<||g")
- for diff_file in ${diff_file_name}; do
- diff_file=$(echo "${diff_file}" | sed 's/[ \t]*//g')
- diff_coverage_data1=$(extract_diff_data ${file_name} ${diff_file} "<")
- read file1 stmts1 miss1 cover1 branch1 <<<"$diff_coverage_data1"
- diff_coverage_data2=$(extract_diff_data ${file_name} ${diff_file} ">")
- read file2 stmts2 miss2 cover2 branch2 <<<"$diff_coverage_data2"
- write_compare_details $file1 $stmts1 $miss1 $branch1 $cover1 $stmts2 $miss2 $branch2 $cover2
- done
- elif [[ $(echo $line | grep "[0-9]d[0-9]") ]] && [[ $(cat ${file_name} | grep -A 1 "$line" | grep "<") ]]; then
- diff_lines=$(sed -n "/${line}/,/^[0-9]/p" ${file_name} | grep "<")
- diff_file_name=$(sed -n "/${line}/,/^[0-9]/p" ${file_name} | grep -Po "<.*[a-z,A-Z].*.py" | sed "s|<||g")
- for diff_file in ${diff_file_name}; do
- diff_file=$(echo "${diff_file}" | sed 's/[ \t]*//g')
- diff_coverage_data=$(extract_diff_data ${file_name} ${diff_file} "<")
- read file stmts miss cover branch <<<"$diff_coverage_data"
- write_compare_details $file $stmts $miss $branch $cover "NA" "NA" "NA" "NA"
- done
- fi
- done
- # generate table end
- echo """
- </table>
- </div>
-</body>
-
-</html>""" >>${output_file}
-}
-
-function main {
- generate_html_head
- generate_coverage_summary
-
- if [[ ${coverage_status} = "SUCCESS" ]]; then
- echo """</body></html>""" >>${output_file}
- echo "coverage PASS, no need to compare difference"
- exit 0
- else
- generate_coverage_details
- fi
-}
-
-main
\ No newline at end of file
diff --git a/.azure-pipelines/scripts/ut/run_ut.sh b/.azure-pipelines/scripts/ut/run_ut.sh
index 2a192dbb8..cf3997357 100644
--- a/.azure-pipelines/scripts/ut/run_ut.sh
+++ b/.azure-pipelines/scripts/ut/run_ut.sh
@@ -1,6 +1,8 @@
#!/bin/bash
set -xe
+test_part=$1
+
# install requirements
echo "##[group]set up UT env..."
export TQDM_MININTERVAL=60
@@ -24,12 +26,16 @@ LOG_DIR=/auto-round/log_dir
mkdir -p ${LOG_DIR}
ut_log_name=${LOG_DIR}/ut.log
-find . -name "test*.py" ! -name "*hpu_only*.py" | sed "s,\.\/,python -m pytest --cov=\"${auto_round_path}\" --cov-report term --html=report.html --self-contained-html --cov-report xml:coverage.xml --cov-append -vs --disable-warnings ,g" > run.sh
+# Split test files into 5 parts
+find . -name "test*.py" ! -name "*hpu_only*.py" | sort > all_tests.txt
+total_lines=$(wc -l < all_tests.txt)
+NUM_CHUNKS=5
+chunk_size=$(( (total_lines + NUM_CHUNKS - 1) / NUM_CHUNKS ))
+start_line=$(( (test_part - 1) * chunk_size + 1 ))
+selected_files=$(tail -n +$start_line all_tests.txt | head -n $chunk_size)
+printf '%s\n' "${selected_files}" | sed "s,\.\/,python -m pytest --cov=\"${auto_round_path}\" --cov-report term --html=report.html --self-contained-html --cov-report xml:coverage.xml --cov-append -vs --disable-warnings ,g" > run.sh
cat run.sh
-bash run.sh 2>&1 | tee ${ut_log_name}
-
-cp report.html ${LOG_DIR}/
-cp coverage.xml ${LOG_DIR}/
+bash run.sh 2>&1 | tee "${ut_log_name}"
if [ $(grep -c '== FAILURES ==' ${ut_log_name}) != 0 ] || [ $(grep -c '== ERRORS ==' ${ut_log_name}) != 0 ] || [ $(grep -c ' passed' ${ut_log_name}) == 0 ]; then
echo "##[error]Find errors in pytest case, please check the output..."
@@ -37,6 +43,6 @@ if [ $(grep -c '== FAILURES ==' ${ut_log_name}) != 0 ] || [ $(grep -c '== ERRORS
fi
# if ut pass, collect the coverage file into artifacts
-cp .coverage ${LOG_DIR}/.coverage
+cp .coverage "${LOG_DIR}/.coverage.part${test_part}"
echo "UT finished successfully! "
diff --git a/.azure-pipelines/scripts/ut/run_ut_cuda.sh b/.azure-pipelines/scripts/ut/run_ut_cuda.sh
index 02073e958..1e80cb885 100644
--- a/.azure-pipelines/scripts/ut/run_ut_cuda.sh
+++ b/.azure-pipelines/scripts/ut/run_ut_cuda.sh
@@ -16,7 +16,7 @@ function create_conda_env() {
# create conda env
source activate base
if conda info --envs | grep -q "^$CONDA_ENV_NAME\s"; then conda remove -n ${CONDA_ENV_NAME} --all -y; fi
- conda create -n ${CONDA_ENV_NAME} python=${PYTHON_VERSION} setuptools=69.5.1 -y
+ conda create -n ${CONDA_ENV_NAME} python=${PYTHON_VERSION} setuptools -y
source activate ${CONDA_ENV_NAME}
conda install -c conda-forge git gxx=11.2.0 gcc=11.2.0 gdb sysroot_linux-64 libgcc uv -y
export LD_PRELOAD=${CONDA_PREFIX}/lib/libstdc++.so.6
@@ -43,6 +43,7 @@ function run_unit_test() {
uv pip install -v git+https://github.com/casper-hansen/AutoAWQ.git --no-build-isolation
uv pip install -v git+https://github.com/ModelCloud/[email protected] --no-build-isolation
+ uv pip install -r https://raw.githubusercontent.com/ModelCloud/GPTQModel/refs/heads/main/requirements.txt
CMAKE_ARGS="-DGGML_CUDA=on -DLLAVA_BUILD=off" uv pip install llama-cpp-python
uv pip install 'git+https://github.com/ggml-org/llama.cpp.git#subdirectory=gguf-py'
uv pip install -r requirements.txt
@@ -105,8 +106,8 @@ function run_unit_test_vlm() {
}
function main() {
- run_unit_test
run_unit_test_vlm
+ run_unit_test
}
main
diff --git a/.azure-pipelines/template/docker-template.yml b/.azure-pipelines/template/docker-template.yml
index 96a078a4b..32bda3d59 100644
--- a/.azure-pipelines/template/docker-template.yml
+++ b/.azure-pipelines/template/docker-template.yml
@@ -60,7 +60,7 @@ steps:
- ${{ if eq(parameters.imageSource, 'build') }}:
- script: |
- docker image prune -a -f
+ docker image prune -f
if [[ ! $(docker images | grep -i ${{ parameters.repoName }}:${{ parameters.repoTag }}) ]]; then
docker build --build-arg USER_ID=$(id -u) --build-arg GROUP_ID=$(id -g) \
-f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/${{parameters.dockerFileName}}.devel -t ${{ parameters.repoName }}:${{ parameters.repoTag }} .
diff --git a/.azure-pipelines/template/ut-template.yml b/.azure-pipelines/template/ut-template.yml
index d2725b523..3962f072d 100644
--- a/.azure-pipelines/template/ut-template.yml
+++ b/.azure-pipelines/template/ut-template.yml
@@ -61,21 +61,11 @@ steps:
artifact: ${{ parameters.utArtifact }}_coverage
publishLocation: "pipeline"
- - task: UseDotNet@2
- displayName: 'Use .NET Core sdk 7.0.x'
- inputs:
- version: 7.0.x
-
- - task: PublishCodeCoverageResults@2
- inputs:
- summaryFileLocation: ${{ parameters.uploadPath }}/coverage.xml
-
- task: Bash@3
condition: always()
inputs:
targetType: "inline"
script: |
- docker exec ${{ parameters.utContainerName }} bash -c "rm -rf /auto-round/* && rm -rf /auto-round/.* || true"
docker stop ${{ parameters.utContainerName }}
docker rm -vf ${{ parameters.utContainerName }} || true
displayName: "Docker clean up"
diff --git a/.azure-pipelines/unit-test-hpu.yml b/.azure-pipelines/unit-test-hpu.yml
index 8eb4c6999..da1e2a6ff 100644
--- a/.azure-pipelines/unit-test-hpu.yml
+++ b/.azure-pipelines/unit-test-hpu.yml
@@ -47,18 +47,11 @@ stages:
uploadPath: $(UPLOAD_PATH)
utArtifact: "ut"
- # - stage: Unit_test_baseline
- # displayName: Unit Test Baseline
- # dependsOn: []
- # jobs:
- # - job:
- # displayName: Unit Test
- # steps:
- # - template: template/ut-template.yml
- # parameters:
- # imageSource: "pull"
- # dockerConfigName: "gitCloneDockerConfig"
- # utScriptFileName: "run_ut_hpu"
- # uploadPath: $(UPLOAD_PATH)
- # utArtifact: "ut_baseline"
- # repo: $(REPO)
+ - task: UseDotNet@2
+ displayName: 'Use .NET Core sdk 7.0.x'
+ inputs:
+ version: 7.0.x
+
+ - task: PublishCodeCoverageResults@2
+ inputs:
+ summaryFileLocation: $(UPLOAD_PATH)/coverage.xml
diff --git a/.azure-pipelines/unit-test.yml b/.azure-pipelines/unit-test.yml
index c4cc085cb..6a3d1338a 100644
--- a/.azure-pipelines/unit-test.yml
+++ b/.azure-pipelines/unit-test.yml
@@ -39,82 +39,78 @@ stages:
dependsOn: []
jobs:
- job:
- displayName: Unit Test
- timeoutInMinutes: 240
+ timeoutInMinutes: 120
+ strategy:
+ matrix:
+ part1:
+ PART: 1
+ part2:
+ PART: 2
+ part3:
+ PART: 3
+ part4:
+ PART: 4
+ part5:
+ PART: 5
steps:
- template: template/ut-template.yml
parameters:
dockerConfigName: "commonDockerConfig"
utScriptFileName: "run_ut"
uploadPath: $(UPLOAD_PATH)
- utArtifact: "ut"
+ utArtifact: "ut-$(PART)"
+ utTestMode: $(PART)
- # - stage: Unit_test_baseline
- # displayName: Unit Test Baseline
- # dependsOn: []
- # jobs:
- # - job:
- # displayName: Unit Test
- # timeoutInMinutes: 120
- # steps:
- # - template: template/ut-template.yml
- # parameters:
- # dockerConfigName: "gitCloneDockerConfig"
- # utScriptFileName: "run_ut"
- # uploadPath: $(UPLOAD_PATH)
- # utArtifact: "ut_baseline"
- # repo: $(REPO)
+ - stage: Coverage
+ displayName: "Collect Coverage"
+ pool:
+ vmImage: "ubuntu-latest"
+ dependsOn: [Unit_test]
+ jobs:
+ - job: CollectDatafiles
+ steps:
+ - script: |
+ docker build --build-arg USER_ID=$(id -u) --build-arg GROUP_ID=$(id -g) -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} .
+ docker images | grep -i ${IMAGE_NAME}
+ if [[ $? -ne 0 ]]; then
+ echo "NO Such Repo"
+ exit 1
+ fi
+ displayName: "Build develop docker image"
- # - stage: Coverage
- # displayName: "Coverage Compare"
- # pool:
- # vmImage: "ubuntu-latest"
- # dependsOn: [Unit_test, Unit_test_baseline]
- # jobs:
- # - job: CollectDatafiles
- # steps:
- # - script: |
- # if [[ ! $(docker images | grep -i ${IMAGE_NAME}:${IMAGE_TAG}) ]]; then
- # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} .
- # fi
- # docker images | grep -i ${IMAGE_NAME}
- # if [[ $? -ne 0 ]]; then
- # echo "NO Such Repo"
- # exit 1
- # fi
- # displayName: "Build develop docker image"
+ - task: DownloadPipelineArtifact@2
+ inputs:
+ artifact:
+ patterns: '*_coverage/.coverage.*'
+ path: $(DOWNLOAD_PATH)
- # - task: DownloadPipelineArtifact@2
- # inputs:
- # artifact:
- # patterns: "*_coverage/.coverage"
- # path: $(DOWNLOAD_PATH)
+ - script: |
+ echo "--- create container ---"
+ docker run -d -it --name="collectLogs" -v ${BUILD_SOURCESDIRECTORY}:/auto-round ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash
+ echo "--- docker ps ---"
+ docker ps
+ echo "--- collect logs ---"
+ docker exec collectLogs bash -c "cd /auto-round \
+ && pip install -r requirements.txt --extra-index-url https://download.pytorch.org/whl/cpu \
+ && pip install -r requirements-cpu.txt \
+ && pip install --no-build-isolation . \
+ && pip list"
+ docker exec collectLogs /bin/bash +x -c "cd /auto-round/.azure-pipelines/scripts \
+ && bash ut/collect_log.sh"
+ displayName: "Collect UT Coverage"
- # - script: |
- # echo "--- create container ---"
- # docker run -d -it --name="collectLogs" -v ${BUILD_SOURCESDIRECTORY}:/auto-round ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash
- # echo "--- docker ps ---"
- # docker ps
- # echo "--- collect logs ---"
- # docker exec collectLogs bash -c "cd /auto-round \
- # && pip install -r requirements.txt \
- # && pip install -vvv --no-build-isolation -e .[cpu] \
- # && pip list"
- # docker exec collectLogs /bin/bash +x -c "cd /auto-round/.azure-pipelines/scripts \
- # && bash ut/collect_log.sh"
- # displayName: "Collect UT Coverage"
+ - task: PublishPipelineArtifact@1
+ condition: succeededOrFailed()
+ inputs:
+ targetPath: $(UPLOAD_PATH)/coverage_PR
+ artifact: $(ARTIFACT_NAME)
+ publishLocation: "pipeline"
- # - task: PublishPipelineArtifact@1
- # condition: succeededOrFailed()
- # inputs:
- # targetPath: $(UPLOAD_PATH)
- # artifact: $(ARTIFACT_NAME)
- # publishLocation: "pipeline"
+ - task: UseDotNet@2
+ displayName: 'Use .NET Core sdk 7.0.x'
+ inputs:
+ version: 7.0.x
- # - task: Bash@3
- # condition: always()
- # inputs:
- # targetType: "inline"
- # script: |
- # docker exec collectLogs bash -c "rm -fr /auto-round/* && rm -fr /auto-round/.* || true"
- # displayName: "Docker clean up"
+ - task: PublishCodeCoverageResults@2
+ inputs:
+ summaryFileLocation: $(Build.SourcesDirectory)/log_dir/coverage_PR/coverage.xml
diff --git a/test/test_cuda/requirements.txt b/test/test_cuda/requirements.txt
index 88b031217..a1f11760c 100644
--- a/test/test_cuda/requirements.txt
+++ b/test/test_cuda/requirements.txt
@@ -7,7 +7,7 @@ einops
# gptqmodel>=2.0
# pip install -v git+https://github.com/ModelCloud/[email protected] --no-build-isolation
intel-extension-for-pytorch
-lm-eval>=0.4.2,<0.5
+lm-eval>=0.4.9.1
numpy < 2.0
optimum
pandas