Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Refactor unit test scripts (#2148)

Sun, Xuehao506bd5fa3078
diff --git a/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh b/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh
index 80fa680db..9125451a1 100644
--- a/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh
+++ b/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh
@@ -32,20 +32,8 @@ function setup_environment() {
}
function print_summary() {
- local status=0
- while IFS= read -r line; do
- if [[ "$line" == *"FAILED"* ]]; then
- $LIGHT_RED && echo "$line" && $RESET
- status=1
- elif [[ "$line" == *"PASSED"* ]]; then
- $LIGHT_GREEN && echo "$line" && $RESET
- elif [[ "$line" == *"NO_TESTS"* ]]; then
- $LIGHT_YELLOW && echo "$line" && $RESET
- else
- echo "$line"
- fi
- done < "${SUMMARY_LOG}"
- exit $status
+ python ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/print_summary.py --summary-log "${SUMMARY_LOG}"
+ exit $?
}
function check_storage_usage() {
@@ -76,7 +64,7 @@ function run_unit_test() {
echo "##[endgroup]"
uv pip list
- export COVERAGE_RCFILE="${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/.coverage"
+ export COVERAGE_RCFILE="${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/.coveragerc"
cd "${BUILD_SOURCESDIRECTORY}/test" || exit 1
@@ -100,9 +88,9 @@ function run_unit_test() {
local test_basename=$(basename ${test_file} .py)
local ut_log_name=${LOG_DIR}/unittest_cuda_${test_basename}.log
- COVERAGE_CORE=sysmon pytest -m "not skip_ci" \
+ pytest -m "not skip_ci" \
--cov=auto_round --cov-report= --cov-append --timeout=60 --session-timeout=720 \
- -vs --disable-warnings --durations=0 --durations-min=1 --junitxml="${ut_log_name%.log}.xml" \
+ -vs --junitxml="${ut_log_name%.log}.xml" \
${test_file} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
done
@@ -128,15 +116,15 @@ function run_unit_test_llmc() {
cd "${BUILD_SOURCESDIRECTORY}/test" || exit 1
- export COVERAGE_RCFILE="${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/.coverage"
+ export COVERAGE_RCFILE="${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/.coveragerc"
for test_file in $(find ./integration/test_cuda -name "test_llmc*.py" | sort); do
echo "##[group]Running ${test_file}..."
local test_basename=$(basename ${test_file} .py)
local ut_log_name=${LOG_DIR}/unittest_cuda_llmc_${test_basename}.log
- COVERAGE_CORE=sysmon pytest -m "not skip_ci" \
- --cov=auto_round --cov-report= --cov-append -vs --disable-warnings \
- --durations=0 --durations-min=1 --junitxml="${ut_log_name%.log}.xml" \
+ pytest -m "not skip_ci" \
+ --cov=auto_round --cov-report= --cov-append -vs \
+ --junitxml="${ut_log_name%.log}.xml" \
${test_file} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
done
@@ -162,15 +150,15 @@ function run_unit_test_sglang() {
echo "##[endgroup]"
cd "${BUILD_SOURCESDIRECTORY}/test" || exit 1
- export COVERAGE_RCFILE="${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/.coverage"
+ export COVERAGE_RCFILE="${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/.coveragerc"
for test_file in $(find ./integration/test_cuda ./e2e/test_cuda -name "test_sglang*.py" | sort); do
echo "##[group]Running ${test_file}..."
local test_basename=$(basename ${test_file} .py)
local ut_log_name=${LOG_DIR}/unittest_cuda_sglang_${test_basename}.log
- COVERAGE_CORE=sysmon pytest -m "not skip_ci" \
- --cov=auto_round --cov-report= --cov-append -vs --disable-warnings \
- --durations=0 --durations-min=1 --junitxml="${ut_log_name%.log}.xml" \
+ pytest -m "not skip_ci" \
+ --cov=auto_round --cov-report= --cov-append -vs \
+ --junitxml="${ut_log_name%.log}.xml" \
${test_file} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
done
@@ -196,15 +184,15 @@ function run_unit_test_vllm() {
echo "##[endgroup]"
cd "${BUILD_SOURCESDIRECTORY}/test" || exit 1
- export COVERAGE_RCFILE="${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/.coverage"
+ export COVERAGE_RCFILE="${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/.coveragerc"
for test_file in $(find ./integration/test_cuda ./e2e/test_cuda -name "test_vllm*.py" | sort); do
echo "##[group]Running ${test_file}..."
local test_basename=$(basename ${test_file} .py)
local ut_log_name=${LOG_DIR}/unittest_cuda_vllm_${test_basename}.log
- COVERAGE_CORE=sysmon pytest -m "not skip_ci" \
- --cov=auto_round --cov-report= --cov-append -vs --disable-warnings \
- --durations=0 --durations-min=1 --junitxml="${ut_log_name%.log}.xml" \
+ pytest -m "not skip_ci" \
+ --cov=auto_round --cov-report= --cov-append -vs \
+ --junitxml="${ut_log_name%.log}.xml" \
${test_file} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
done
diff --git a/.azure-pipelines/scripts/ut/.coverage b/.azure-pipelines/scripts/ut/.coveragerc
similarity index 88%
rename from .azure-pipelines/scripts/ut/.coverage
rename to .azure-pipelines/scripts/ut/.coveragerc
index 28b2c2c9a..97e602dfa 100644
--- a/.azure-pipelines/scripts/ut/.coverage
+++ b/.azure-pipelines/scripts/ut/.coveragerc
@@ -1,5 +1,5 @@
[run]
-branch = True
+core = sysmon
[paths]
source =
diff --git a/.azure-pipelines/scripts/ut/collect_log.sh b/.azure-pipelines/scripts/ut/collect_log.sh
index 5e26c8240..d6a7828d5 100644
--- a/.azure-pipelines/scripts/ut/collect_log.sh
+++ b/.azure-pipelines/scripts/ut/collect_log.sh
@@ -1,7 +1,7 @@
#!/bin/bash
set -e
uv pip install coverage
-export COVERAGE_RCFILE=${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/.coverage
+export COVERAGE_RCFILE=${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/.coveragerc
coverage_log="${BUILD_SOURCESDIRECTORY}/log_dir/coverage_log"
cd "${BUILD_SOURCESDIRECTORY}/log_dir"
diff --git a/.azure-pipelines/scripts/ut/print_summary.py b/.azure-pipelines/scripts/ut/print_summary.py
new file mode 100644
index 000000000..36ec2696d
--- /dev/null
+++ b/.azure-pipelines/scripts/ut/print_summary.py
@@ -0,0 +1,47 @@
+"""Print a colorized test summary and exit non-zero on any failure.
+
+Reads the summary log line by line and prints each line with an ANSI color
+based on its content:
+
+* lines containing ``FAILED`` -> red (and force a non-zero exit code)
+* lines containing ``PASSED`` -> green
+* lines containing ``NO_TESTS`` -> yellow
+* everything else -> unchanged
+"""
+
+import argparse
+import sys
+
+RED = "\033[31m"
+GREEN = "\033[32m"
+YELLOW = "\033[33m"
+RESET = "\033[0m"
+
+
+def print_summary(summary_log: str) -> int:
+ """Print the colorized summary and return the exit status."""
+ status = 0
+ with open(summary_log, encoding="utf-8") as f:
+ for line in f:
+ line = line.rstrip("\n")
+ if "FAILED" in line:
+ print(f"{RED}{line}{RESET}")
+ status = 1
+ elif "PASSED" in line:
+ print(f"{GREEN}{line}{RESET}")
+ elif "NO_TESTS" in line:
+ print(f"{YELLOW}{line}{RESET}")
+ else:
+ print(line)
+ return status
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--summary-log", required=True, help="Path to the summary log file.")
+ args = parser.parse_args()
+ sys.exit(print_summary(args.summary_log))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/.azure-pipelines/scripts/ut/run_ut.sh b/.azure-pipelines/scripts/ut/run_ut.sh
index 67aaec68d..14e4db8cf 100644
--- a/.azure-pipelines/scripts/ut/run_ut.sh
+++ b/.azure-pipelines/scripts/ut/run_ut.sh
@@ -31,27 +31,15 @@ function setup_environment() {
export LD_LIBRARY_PATH=${HOME}/.venv/lib/:$LD_LIBRARY_PATH
export FORCE_BF16=1
- export COVERAGE_RCFILE=/auto-round/.azure-pipelines/scripts/ut/.coverage
+ export COVERAGE_RCFILE=/auto-round/.azure-pipelines/scripts/ut/.coveragerc
echo "##[endgroup]"
uv pip list
}
function print_summary() {
- local status=0
- while IFS= read -r line; do
- if [[ "$line" == *"FAILED"* ]]; then
- $LIGHT_RED && echo "$line" && $RESET
- status=1
- elif [[ "$line" == *"PASSED"* ]]; then
- $LIGHT_GREEN && echo "$line" && $RESET
- elif [[ "$line" == *"NO_TESTS"* ]]; then
- $LIGHT_YELLOW && echo "$line" && $RESET
- else
- echo "$line"
- fi
- done < "${SUMMARY_LOG}"
- exit $status
+ python /auto-round/.azure-pipelines/scripts/ut/print_summary.py --summary-log "${SUMMARY_LOG}"
+ exit $?
}
function check_storage_usage() {
@@ -90,10 +78,10 @@ function run_unit_test() {
local test_basename=$(basename ${test_file} .py)
local ut_log_name=${LOG_DIR}/unittest_${test_basename}.log
- COVERAGE_CORE=sysmon numactl --physcpubind="${NUMA_CPUSET:-0-15}" --membind="${NUMA_NODE:-0}" \
- pytest -m "not skip_ci" --timeout=30 --session-timeout=600 --durations=0 --durations-min=1 \
+ numactl --physcpubind="${NUMA_CPUSET:-0-15}" --membind="${NUMA_NODE:-0}" \
+ pytest -m "not skip_ci" --timeout=30 --session-timeout=600 \
--cov=auto_round --cov-report= --cov-append \
- -vs --disable-warnings --junitxml="${ut_log_name%.log}.xml" ${test_file} 2>&1 | tee ${ut_log_name}
+ -vs --junitxml="${ut_log_name%.log}.xml" ${test_file} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
done
}
@@ -110,10 +98,10 @@ function run_inc_unit_test() {
local test_basename=$(basename ${test_file} .py)
local ut_log_name=${LOG_DIR}/unittest_${test_basename}.log
- COVERAGE_CORE=sysmon numactl --physcpubind="${NUMA_CPUSET:-0-15}" --membind="${NUMA_NODE:-0}" \
- pytest --timeout=30 --session-timeout=600 --durations=0 --durations-min=1 \
+ numactl --physcpubind="${NUMA_CPUSET:-0-15}" --membind="${NUMA_NODE:-0}" \
+ pytest --timeout=30 --session-timeout=600 \
--cov=auto_round --cov-report= --cov-append \
- -vs --disable-warnings --junitxml="${ut_log_name%.log}.xml" ${test_file} 2>&1 | tee ${ut_log_name}
+ -vs --junitxml="${ut_log_name%.log}.xml" ${test_file} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
done
}
@@ -132,10 +120,10 @@ function run_llmc_unit_test() {
local test_basename=$(basename ${test_file} .py)
local ut_log_name=${LOG_DIR}/unittest_${test_basename}.log
- COVERAGE_CORE=sysmon numactl --physcpubind="${NUMA_CPUSET:-0-15}" --membind="${NUMA_NODE:-0}" \
- pytest --timeout=30 --session-timeout=600 --durations=0 --durations-min=1 \
+ numactl --physcpubind="${NUMA_CPUSET:-0-15}" --membind="${NUMA_NODE:-0}" \
+ pytest --timeout=30 --session-timeout=600 \
--cov=auto_round --cov-report= --cov-append \
- -vs --disable-warnings --junitxml="${ut_log_name%.log}.xml" ${test_file} 2>&1 | tee ${ut_log_name}
+ -vs --junitxml="${ut_log_name%.log}.xml" ${test_file} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
done
}
diff --git a/.azure-pipelines/scripts/ut/run_ut_cuda.sh b/.azure-pipelines/scripts/ut/run_ut_cuda.sh
index e9941c933..587495831 100644
--- a/.azure-pipelines/scripts/ut/run_ut_cuda.sh
+++ b/.azure-pipelines/scripts/ut/run_ut_cuda.sh
@@ -95,15 +95,15 @@ function run_unit_test() {
uv pip install torch==2.13.0 torchvision torchao --index-url https://download.pytorch.org/whl/cu130
uv pip install llama-cpp-python --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu130
uv pip install 'git+https://github.com/ggml-org/llama.cpp.git#subdirectory=gguf-py'
- uv pip install -r test/unit/test_cuda/requirements.txt
- uv pip install -r test/unit/test_cuda/requirements_diffusion.txt
+ uv pip install -r unit/test_cuda/requirements.txt
+ uv pip install -r unit/test_cuda/requirements_diffusion.txt
uv pip install -U transformers chardet
uv pip uninstall torch torchvision
uv pip install torch==2.13.0 torchvision torchao --index-url https://download.pytorch.org/whl/cu130
cd ${REPO_PATH} && uv pip install . && cd ${REPO_PATH}/test
pip list > ${LOG_DIR}/ut_pip_list.txt
- export COVERAGE_RCFILE=${REPO_PATH}/.azure-pipelines/scripts/ut/.coverage
+ export COVERAGE_RCFILE=${REPO_PATH}/.azure-pipelines/scripts/ut/.coveragerc
# run unit tests individually with separate logs
for test_file in $(find ./unit/test_cuda -type f -name "test_*.py" | grep -Ev "vlms|llmc|sglang|vllm|multiple_card" | sort); do
@@ -111,8 +111,8 @@ function run_unit_test() {
local ut_log_name=${LOG_DIR}/unittest_cuda_${test_basename}.log
echo "Running ${test_file}..."
- COVERAGE_CORE=sysmon pytest --cov=auto_round --cov-report= --cov-append \
- --durations=0 --durations-min=1 -vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}
+ pytest --cov=auto_round --cov-report= --cov-append \
+ -p no:timeout -vs ${test_file} 2>&1 | tee ${ut_log_name}
done
[ -f .coverage ] && cp .coverage ${LOG_DIR}/.coverage.unit
@@ -131,14 +131,14 @@ function run_unit_test_vlm() {
uv pip install torch==2.13.0 torchvision --index-url https://download.pytorch.org/whl/cu130
uv pip install git+https://github.com/haotian-liu/[email protected] --no-deps
uv pip install flash-attn==2.8.3 --no-build-isolation
- uv pip install -r test_cuda/requirements_vlm.txt \
+ uv pip install -r unit/test_cuda/requirements_vlm.txt \
--extra-index-url https://download.pytorch.org/whl/cu130 \
--index-strategy unsafe-best-match
uv pip install -U chardet
cd ${REPO_PATH} && uv pip install . && cd ${REPO_PATH}/test
pip list > ${LOG_DIR}/vlm_ut_pip_list.txt
- export COVERAGE_RCFILE=${REPO_PATH}/.azure-pipelines/scripts/ut/.coverage
+ export COVERAGE_RCFILE=${REPO_PATH}/.azure-pipelines/scripts/ut/.coveragerc
# run VLM unit tests individually with separate logs
for test_file in $(find ./unit/test_cuda -name "test*vlms.py"); do
@@ -146,8 +146,8 @@ function run_unit_test_vlm() {
local ut_log_name=${LOG_DIR}/unittest_cuda_vlm_${test_basename}.log
echo "Running ${test_file}..."
- COVERAGE_CORE=sysmon pytest --cov=auto_round --cov-report= --cov-append \
- --durations=0 --durations-min=1 -vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}
+ pytest --cov=auto_round --cov-report= --cov-append \
+ -p no:timeout -vs ${test_file} 2>&1 | tee ${ut_log_name}
done
[ -f .coverage ] && cp .coverage ${LOG_DIR}/.coverage.vlm
@@ -163,12 +163,14 @@ function run_unit_test_llmc() {
cd ${REPO_PATH}/test
rm -rf .coverage* *.xml *.html
- BUILD_TYPE="nightly" uv pip install -r test_cuda/requirements_llmc.txt --extra-index-url https://download.pytorch.org/whl/cu130 --index-strategy unsafe-best-match
+ BUILD_TYPE="nightly" uv pip install -r integration/test_cuda/requirements_llmc.txt \
+ --extra-index-url https://download.pytorch.org/whl/cu130 \
+ --index-strategy unsafe-best-match
uv pip install -U chardet
cd ${REPO_PATH} && uv pip install . && cd ${REPO_PATH}/test
pip list > ${LOG_DIR}/llmc_ut_pip_list.txt
- export COVERAGE_RCFILE=${REPO_PATH}/.azure-pipelines/scripts/ut/.coverage
+ export COVERAGE_RCFILE=${REPO_PATH}/.azure-pipelines/scripts/ut/.coveragerc
# run unit tests individually with separate logs
for test_file in $(find ./integration/test_cuda -name "test_llmc*.py" | sort); do
@@ -176,8 +178,8 @@ function run_unit_test_llmc() {
local ut_log_name=${LOG_DIR}/unittest_cuda_llmc_${test_basename}.log
echo "Running ${test_file}..."
- COVERAGE_CORE=sysmon pytest --cov=auto_round --cov-report= --cov-append \
- --durations=0 --durations-min=1 -vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}
+ pytest --cov=auto_round --cov-report= --cov-append \
+ -p no:timeout -vs ${test_file} 2>&1 | tee ${ut_log_name}
done
[ -f .coverage ] && cp .coverage ${LOG_DIR}/.coverage.llmc
@@ -193,7 +195,7 @@ function run_unit_test_sglang() {
cd ${REPO_PATH}/test
rm -rf .coverage* *.xml *.html
- uv pip install -r test_cuda/requirements_sglang.txt \
+ uv pip install -r integration/test_cuda/requirements_sglang.txt \
--prerelease=allow \
--extra-index-url https://download.pytorch.org/whl/cu130 \
--index-strategy unsafe-best-match
@@ -202,7 +204,7 @@ function run_unit_test_sglang() {
cd ${REPO_PATH} && uv pip install . && cd ${REPO_PATH}/test
pip list > ${LOG_DIR}/sglang_ut_pip_list.txt
- export COVERAGE_RCFILE=${REPO_PATH}/.azure-pipelines/scripts/ut/.coverage
+ export COVERAGE_RCFILE=${REPO_PATH}/.azure-pipelines/scripts/ut/.coveragerc
# run unit tests individually with separate logs
for test_file in $(find ./integration/test_cuda ./e2e/test_cuda -name "test_sglang*.py" | sort); do
@@ -210,8 +212,8 @@ function run_unit_test_sglang() {
local ut_log_name=${LOG_DIR}/unittest_cuda_sglang_${test_basename}.log
echo "Running ${test_file}..."
- COVERAGE_CORE=sysmon pytest --cov=auto_round --cov-report= --cov-append \
- --durations=0 --durations-min=1 -vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}
+ pytest --cov=auto_round --cov-report= --cov-append \
+ -p no:timeout -vs ${test_file} 2>&1 | tee ${ut_log_name}
done
[ -f .coverage ] && cp .coverage ${LOG_DIR}/.coverage.sglang
@@ -228,7 +230,7 @@ function run_unit_test_vllm() {
cd ${REPO_PATH}/test
rm -rf .coverage* *.xml *.html
- uv pip install -r test_cuda/requirements_vllm.txt \
+ uv pip install -r integration/test_cuda/requirements_vllm.txt \
--extra-index-url https://download.pytorch.org/whl/cu130 \
--index-strategy unsafe-best-match
local flashinfer_version=$(uv pip show flashinfer-python 2>/dev/null | grep -i "^Version" | awk '{print $2}')
@@ -237,7 +239,7 @@ function run_unit_test_vllm() {
cd ${REPO_PATH} && uv pip install . && cd ${REPO_PATH}/test
pip list > ${LOG_DIR}/vllm_ut_pip_list.txt
- export COVERAGE_RCFILE=${REPO_PATH}/.azure-pipelines/scripts/ut/.coverage
+ export COVERAGE_RCFILE=${REPO_PATH}/.azure-pipelines/scripts/ut/.coveragerc
# run unit tests individually with separate logs
for test_file in $(find ./integration/test_cuda ./e2e/test_cuda -name "test_vllm*.py" | sort); do
@@ -245,8 +247,8 @@ function run_unit_test_vllm() {
local ut_log_name=${LOG_DIR}/unittest_cuda_vllm_${test_basename}.log
echo "Running ${test_file}..."
- COVERAGE_CORE=sysmon pytest --cov=auto_round --cov-report= --cov-append \
- --durations=0 --durations-min=1 -vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}
+ pytest --cov=auto_round --cov-report= --cov-append \
+ -p no:timeout -vs ${test_file} 2>&1 | tee ${ut_log_name}
done
[ -f .coverage ] && cp .coverage ${LOG_DIR}/.coverage.vllm
@@ -266,7 +268,7 @@ function merge_coverage() {
return
fi
- export COVERAGE_RCFILE=${REPO_PATH}/.azure-pipelines/scripts/ut/.coverage
+ export COVERAGE_RCFILE=${REPO_PATH}/.azure-pipelines/scripts/ut/.coveragerc
coverage combine ${coverage_files}
coverage xml -o ${LOG_DIR}/coverage_merged.xml
coverage html -d ${LOG_DIR}/htmlcov
diff --git a/.azure-pipelines/scripts/ut/run_ut_hpu.sh b/.azure-pipelines/scripts/ut/run_ut_hpu.sh
index 67e5b9149..77f88d346 100644
--- a/.azure-pipelines/scripts/ut/run_ut_hpu.sh
+++ b/.azure-pipelines/scripts/ut/run_ut_hpu.sh
@@ -17,7 +17,7 @@ function setup_environment() {
export LD_LIBRARY_PATH=/usr/local/lib/:$LD_LIBRARY_PATH
export FORCE_BF16=1
- export COVERAGE_RCFILE=/auto-round/.azure-pipelines/scripts/ut/.coverage
+ export COVERAGE_RCFILE=/auto-round/.azure-pipelines/scripts/ut/.coveragerc
LOG_DIR=/auto-round/log_dir
mkdir -p ${LOG_DIR}
@@ -33,37 +33,25 @@ function run_unit_test() {
echo "##[group]Running ${test_file} in HPU lazy mode..."
local ut_log_name="${LOG_DIR}/unittest_lazy_${test_basename}.log"
- COVERAGE_CORE=sysmon PT_HPU_LAZY_MODE=1 pytest --cov="${auto_round_path}" \
- --timeout=30 --session-timeout=600 --durations=0 --durations-min=1 \
- --cov-report= --cov-append -vs --disable-warnings \
+ PT_HPU_LAZY_MODE=1 pytest --cov="${auto_round_path}" \
+ --timeout=30 --session-timeout=600 \
+ --cov-report= --cov-append -vs \
--junitxml="${ut_log_name%.log}.xml" ${test_file} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
echo "##[group]Running ${test_file} in HPU compile mode..."
local ut_log_name="${LOG_DIR}/unittest_compile_${test_basename}.log"
- COVERAGE_CORE=sysmon PT_HPU_LAZY_MODE=0 pytest --mode compile --cov="${auto_round_path}" \
- --timeout=30 --session-timeout=600 --durations=0 --durations-min=1 \
- --cov-report= --cov-append -vs --disable-warnings \
+ PT_HPU_LAZY_MODE=0 pytest --mode compile --cov="${auto_round_path}" \
+ --timeout=30 --session-timeout=600 \
+ --cov-report= --cov-append -vs \
--junitxml="${ut_log_name%.log}.xml" ${test_file} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
done
}
function print_summary() {
- local status=0
- while IFS= read -r line; do
- if [[ "$line" == *"FAILED"* ]]; then
- $LIGHT_RED && echo "$line" && $RESET
- status=1
- elif [[ "$line" == *"PASSED"* ]]; then
- $LIGHT_GREEN && echo "$line" && $RESET
- elif [[ "$line" == *"NO_TESTS"* ]]; then
- $LIGHT_YELLOW && echo "$line" && $RESET
- else
- echo "$line"
- fi
- done < "${SUMMARY_LOG}"
- exit $status
+ python /auto-round/.azure-pipelines/scripts/ut/print_summary.py --summary-log "${SUMMARY_LOG}"
+ exit $?
}
function collect_log() {
diff --git a/.azure-pipelines/scripts/ut/run_ut_xpu.sh b/.azure-pipelines/scripts/ut/run_ut_xpu.sh
index d69e6f563..642f1172f 100644
--- a/.azure-pipelines/scripts/ut/run_ut_xpu.sh
+++ b/.azure-pipelines/scripts/ut/run_ut_xpu.sh
@@ -22,7 +22,7 @@ function setup_environment() {
export TQDM_MININTERVAL=60
export HF_HUB_DISABLE_PROGRESS_BARS=1
export LD_LIBRARY_PATH=${HOME}/.venv/lib/:$LD_LIBRARY_PATH
- export COVERAGE_RCFILE=/auto-round/.azure-pipelines/scripts/ut/.coverage
+ export COVERAGE_RCFILE=/auto-round/.azure-pipelines/scripts/ut/.coveragerc
LOG_DIR=/auto-round/log_dir
mkdir -p ${LOG_DIR}
@@ -38,9 +38,9 @@ function run_unit_test() {
echo "##[group]Running ark ${test_file}..."
local ut_log_name="${LOG_DIR}/unittest_ark_${test_basename}.log"
- COVERAGE_CORE=sysmon numactl --physcpubind="${NUMA_CPUSET:-0-27}" --membind="${NUMA_NODE:-0}" \
- pytest --timeout=30 --session-timeout=600 --durations=0 --durations-min=1 \
- --cov="${auto_round_path}" --cov-report= --cov-append -vs --disable-warnings \
+ numactl --physcpubind="${NUMA_CPUSET:-0-27}" --membind="${NUMA_NODE:-0}" \
+ pytest --timeout=30 --session-timeout=600 \
+ --cov="${auto_round_path}" --cov-report= --cov-append -vs \
--junitxml="${ut_log_name%.log}.xml" ${test_file} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
done
@@ -50,9 +50,9 @@ function run_unit_test() {
echo "##[group]Running xpu ${test_file}..."
local ut_log_name="${LOG_DIR}/unittest_xpu_${test_basename}.log"
- COVERAGE_CORE=sysmon numactl --physcpubind="${NUMA_CPUSET:-0-27}" --membind="${NUMA_NODE:-0}" \
- pytest --timeout=30 --session-timeout=600 --durations=0 --durations-min=1 \
- --cov="${auto_round_path}" --cov-report= --cov-append -vs --disable-warnings \
+ numactl --physcpubind="${NUMA_CPUSET:-0-27}" --membind="${NUMA_NODE:-0}" \
+ pytest --timeout=30 --session-timeout=600 \
+ --cov="${auto_round_path}" --cov-report= --cov-append -vs \
--junitxml="${ut_log_name%.log}.xml" ${test_file} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
done
@@ -71,29 +71,17 @@ function run_unit_test_llmc() {
echo "##[group]Running xpu llmc ${test_file}..."
local ut_log_name="${LOG_DIR}/unittest_xpu_${test_basename}.log"
- COVERAGE_CORE=sysmon numactl --physcpubind="${NUMA_CPUSET:-0-27}" --membind="${NUMA_NODE:-0}" \
- pytest --timeout=30 --session-timeout=600 --durations=0 --durations-min=1 \
- --cov="${auto_round_path}" --cov-report= --cov-append -vs --disable-warnings \
+ numactl --physcpubind="${NUMA_CPUSET:-0-27}" --membind="${NUMA_NODE:-0}" \
+ pytest --timeout=30 --session-timeout=600 \
+ --cov="${auto_round_path}" --cov-report= --cov-append -vs \
--junitxml="${ut_log_name%.log}.xml" ${test_file} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
done
}
function print_summary() {
- local status=0
- while IFS= read -r line; do
- if [[ "$line" == *"FAILED"* ]]; then
- $LIGHT_RED && echo "$line" && $RESET
- status=1
- elif [[ "$line" == *"PASSED"* ]]; then
- $LIGHT_GREEN && echo "$line" && $RESET
- elif [[ "$line" == *"NO_TESTS"* ]]; then
- $LIGHT_YELLOW && echo "$line" && $RESET
- else
- echo "$line"
- fi
- done < "${SUMMARY_LOG}"
- exit $status
+ python /auto-round/.azure-pipelines/scripts/ut/print_summary.py --summary-log "${SUMMARY_LOG}"
+ exit $?
}
function collect_log() {
diff --git a/.azure-pipelines/unit-test.yml b/.azure-pipelines/unit-test.yml
index a557ac4d2..eb10a27bd 100644
--- a/.azure-pipelines/unit-test.yml
+++ b/.azure-pipelines/unit-test.yml
@@ -30,8 +30,9 @@ pr:
- test/e2e
- "*.md"
- "**/*.md"
- - .azure-pipelines/scripts/ut/run_ut_hpu.sh
+ - .azure-pipelines/scripts/ut/run_ark_ut.sh
- .azure-pipelines/scripts/ut/run_ut_cuda.sh
+ - .azure-pipelines/scripts/ut/run_ut_hpu.sh
- .azure-pipelines/scripts/ut/run_ut_xpu.sh
pool: INTEL-CPU
@@ -44,6 +45,16 @@ variables:
ARTIFACT_NAME: "UT_coverage_report"
REPO: $(Build.Repository.Uri)
+parameters:
+ - name: unitTestParts
+ type: object
+ default:
+ - 1
+ - 2
+ - 3
+ - 4
+ - 5
+
stages:
- template: template/lib-build-template.yml
parameters:
@@ -60,16 +71,9 @@ stages:
timeoutInMinutes: 60
strategy:
matrix:
- part1:
- PART: 1
- part2:
- PART: 2
- part3:
- PART: 3
- part4:
- PART: 4
- part5:
- PART: 5
+ ${{ each part in parameters.unitTestParts }}:
+ part_${{ part }}:
+ PART: ${{ part }}
steps:
- template: template/ut-template.yml
parameters:
diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.py
index 17422904b..092505898 100644
--- a/auto_round/calib_dataset.py
+++ b/auto_round/calib_dataset.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import itertools
import json
import logging
import multiprocessing
@@ -1015,7 +1016,12 @@ def _get_dataset_impl(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed
if name in data_lens:
dataset = select_dataset(dataset, range(data_lens[name]))
if isinstance(dataset, IterableDataset):
- dataset = Dataset.from_list(list(dataset))
+ # A single dataset source never contributes more than `nsamples` rows to the
+ # final combined dataset, so cap materialization here instead of fully consuming
+ # the (much larger) internal `.take(...)` pool used by streaming dataset loaders.
+ # This avoids needless downloads/shard resolution for large remote datasets
+ # (e.g. BAAI/CCI3-HQ) when only a small subset of samples is actually needed.
+ dataset = Dataset.from_list(list(itertools.islice(dataset, nsamples)))
dataset.set_format(type="torch", columns=["input_ids", "attention_mask"])
new_features = {}
for k, v in dataset.features.items():
diff --git a/test/integration/test_cpu/test_llmc_integration.py b/test/integration/test_cpu/test_llmc_integration.py
index c297dc95a..c2dbfdbc6 100644
--- a/test/integration/test_cpu/test_llmc_integration.py
+++ b/test/integration/test_cpu/test_llmc_integration.py
@@ -10,7 +10,7 @@ from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
from auto_round import AutoRound, AWQConfig
from auto_round.calib_dataset import get_dataset
-from ...envs import multi_card
+from ...unit.envs import multi_card
recipe_str = """
quant_stage:
diff --git a/test/integration/test_cuda/test_llmc_integration.py b/test/integration/test_cuda/test_llmc_integration.py
index 3422e3cdc..ef860e8d6 120000
--- a/test/integration/test_cuda/test_llmc_integration.py
+++ b/test/integration/test_cuda/test_llmc_integration.py
@@ -1 +1 @@
-../../test_cpu/integrations/test_llmc_integration.py
\ No newline at end of file
+../test_cpu/test_llmc_integration.py
\ No newline at end of file
diff --git a/test/pytest.ini b/test/pytest.ini
index a2e9b4e33..6e41873ea 100644
--- a/test/pytest.ini
+++ b/test/pytest.ini
@@ -3,4 +3,4 @@ testpaths = unit
python_files = test_*.py
python_classes = Test*
python_functions = test_*
-addopts = -v --tb=short --ignore=test/unit/test_xpu
+addopts = -vs --tb=short --ignore=test/unit/test_xpu --durations=0 --durations-min=1 --disable-warnings