PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Enhance test logging and retry mechanism for unit tests (#2274)
chen, suyueb48c9db6744e
diff --git a/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh b/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.shindex cf05383b6..dd1b83167 100644--- a/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh+++ b/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh@@ -26,6 +26,8 @@ source ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/detect_changed_testLOG_DIR="${BUILD_SOURCESDIRECTORY}/log_dir"mkdir -p "${LOG_DIR}"++source ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/retry_failed_tests.shSUMMARY_LOG="${LOG_DIR}/results_summary.log"# print_summary reads this file unconditionally; a matrix part that selects no# test never writes it, so make sure it always exists.@@ -69,9 +71,11 @@ function setup_basic_test_env() {uv pip uninstall torch torchvisionuv pip install torch==2.13.0 torchvision torchao --index-url https://download.pytorch.org/whl/cu130uv pip install .- echo "##[endgroup]"+ echo "List dependencies ..."uv pip list+ echo "##[endgroup]"+cd "${BUILD_SOURCESDIRECTORY}/test" || exit 1}@@ -80,6 +84,8 @@ function run_pytest() {local ut_log_name=$2echo "##[group]Running ${test_case}..."+ # Record the test targets so a retry can rerun exactly these cases.+ printf '%s\n' ${test_case} > "${ut_log_name%.log}.list"pytest -m "not skip_ci" --cov=auto_round --cov-report= --cov-append -vs \--junitxml="${ut_log_name%.log}.xml" ${test_case} 2>&1 | tee ${ut_log_name}echo "##[endgroup]"@@ -100,6 +106,8 @@ function run_common_group() {}function run_common_unit_test() {+ run_if_retry && return 0+# common test case for cpu/gpu/xpu# Group cases by the first-level folder under unit/common; a single test# file placed directly under unit/common (e.g. test_main.py) runs on its own.@@ -116,6 +124,8 @@ function run_common_unit_test() {function run_unit_test() {+ run_if_retry && return 0+# run ci cuda ut scopefind ./unit/test_cuda -type f -name "test_*.py" | grep -Ev "vlms|llmc|sglang|vllm|multiple_card" | sort > all_tests.txttotal_lines=$(wc -l < all_tests.txt)@@ -221,11 +231,17 @@ function run_unit_test_vllm() {function collect_log() {touch "${SUMMARY_LOG}"+ # collect_result.py also stages only the failed logs so a retry can rerun them.python ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/collect_result.py \- --test-type "Unit Tests" --log-pattern "unittest_cuda_*.log" --log-dir ${LOG_DIR} --summary-log ${SUMMARY_LOG}+ --test-type "Unit Tests" --log-pattern "unittest_*.log" --log-dir ${LOG_DIR} \+ --summary-log ${SUMMARY_LOG} --failed-logs-dir "${LOG_DIR}/failed_logs"if [ -f .coverage ]; thencp .coverage "${LOG_DIR}/.coverage.part${test_part}"+ # Keep .coverage in the failure artifact so a retry can accumulate onto it.+ if [ -d "${LOG_DIR}/failed_logs" ]; then+ cp .coverage "${LOG_DIR}/failed_logs/.coverage"+ fifi}diff --git a/.azure-pipelines/scripts/ut/collect_result.py b/.azure-pipelines/scripts/ut/collect_result.pyindex 58a420a19..0b23ad9e2 100644--- a/.azure-pipelines/scripts/ut/collect_result.py+++ b/.azure-pipelines/scripts/ut/collect_result.py@@ -13,7 +13,10 @@ as FAILED."""import argparse+import os+import shutilimport sys+import tempfileimport xml.etree.ElementTree as ETfrom dataclasses import dataclassfrom enum import Enum, auto@@ -46,6 +49,7 @@ class TestResult:status: TestStatusduration: strcounts: TestCounts+ log_path: Path | None = Noneclass XmlAnalyzer:@@ -89,6 +93,7 @@ class XmlAnalyzer:status=self._determine_status(counts, xml_file),duration=duration,counts=counts if counts is not None else TestCounts(),+ log_path=log_file,)def _extract_test_name(self, path: Path) -> str:@@ -229,12 +234,66 @@ class ReportGenerator:)+def failed_log_paths(results: list[TestResult]) -> list[Path]:+ """Return the log paths of the failed tests."""+ return [r.log_path for r in results if r.status == TestStatus.FAILED and r.log_path and r.log_path.is_file()]+++def stage_failed_logs(log_paths: list[Path], failed_dir: Path) -> None:+ """Copy the failed test logs into ``failed_dir`` and assemble failed_tests.list.++ ``failed_tests.list`` records the test targets of every failed log (read from+ the sibling ``.list`` files) so a clean-environment retry can rerun exactly+ those cases. Centralized here so every hardware UT runner behaves the same.++ ``failed_tests.list`` is written atomically (temp file + rename) so an+ interruption cannot leave a partial list that would shrink the retry scope.+ """+ failed_dir.mkdir(parents=True, exist_ok=True)+ failed_cases: list[str] = []+ seen: set[str] = set()+ for log_path in log_paths:+ shutil.copy2(log_path, failed_dir / log_path.name)+ list_file = log_path.with_suffix(".list")+ if list_file.is_file():+ for case in list_file.read_text(encoding="utf-8").split():+ if case not in seen:+ seen.add(case)+ failed_cases.append(case)+ _atomic_write_text(+ failed_dir / "failed_tests.list",+ "\n".join(failed_cases) + ("\n" if failed_cases else ""),+ )+ print(+ f"Staged {len(log_paths)} failed log(s) and {len(failed_cases)} case(s) " f"into: {failed_dir.absolute()}",+ file=sys.stderr,+ )+++def _atomic_write_text(target: Path, text: str) -> None:+ """Write ``text`` to ``target`` atomically via a temp file in the same dir."""+ fd, tmp_name = tempfile.mkstemp(dir=str(target.parent), prefix=f".{target.name}.", suffix=".tmp")+ try:+ with os.fdopen(fd, "w", encoding="utf-8") as f:+ f.write(text)+ f.flush()+ os.fsync(f.fileno())+ os.replace(tmp_name, target)+ except BaseException:+ try:+ os.unlink(tmp_name)+ except OSError:+ pass+ raise++def main():parser = argparse.ArgumentParser(description="Analyze JUnit XML reports and generate summary")parser.add_argument("--test-type", required=True, help="Type of tests")parser.add_argument("--log-dir", required=True, type=Path, help="Directory with logs/reports")parser.add_argument("--summary-log", required=True, type=Path, help="Output file")parser.add_argument("--log-pattern", default="*.log", help="Glob pattern for log files")+ parser.add_argument("--failed-logs-dir", type=Path, help="Optional dir to stage only the failed test logs into")args = parser.parse_args()@@ -262,6 +321,9 @@ def main():f"{stats['failed']} failed, {stats['skipped']} skipped")+ if args.failed_logs_dir is not None and failed > 0:+ stage_failed_logs(failed_log_paths(results), args.failed_logs_dir)+except Exception as e:print(f"Error: {e}", file=sys.stderr)import tracebackdiff --git a/.azure-pipelines/scripts/ut/detect_changed_tests.sh b/.azure-pipelines/scripts/ut/detect_changed_tests.shindex 8753d4dcb..9c2824eaa 100644--- a/.azure-pipelines/scripts/ut/detect_changed_tests.sh+++ b/.azure-pipelines/scripts/ut/detect_changed_tests.sh@@ -87,8 +87,8 @@ detect_changed_tests() {<(printf '%s\n' "${test_changed}" | sort -u))"if [ -n "${non_test}" ]; thenecho "##[group]non-test files changed, run full suite:" >&2- printf ' %s\n' ${non_test} >&2- echo "##[endgroup]"+ printf ' %s\n' "${non_test}" >&2+ echo "##[endgroup]" >&2return 1fidiff --git a/.azure-pipelines/scripts/ut/retry_failed_tests.sh b/.azure-pipelines/scripts/ut/retry_failed_tests.shnew file mode 100644index 000000000..9b0e3386f--- /dev/null+++ b/.azure-pipelines/scripts/ut/retry_failed_tests.sh@@ -0,0 +1,24 @@+# A retry is any job attempt after the first; failed_tests.list is downloaded+# from the previous attempt before this script runs (clean environment).++SYSTEM_JOBATTEMPT=${SYSTEM_JOBATTEMPT:-1}+LOG_DIR=${LOG_DIR:-/auto-round/log_dir}++function is_retry() {+ [ "${SYSTEM_JOBATTEMPT:-1}" -gt 1 ] && [ -s "${LOG_DIR}/failed_tests.list" ]+}++function retry_selection() {+ is_retry && sort -u "${LOG_DIR}/failed_tests.list"+}++function run_if_retry() {+ is_retry || return 1+ # Seed coverage from the previous attempt so --cov-append accumulates.+ if [ -f "${LOG_DIR}/.coverage" ]; then+ cp "${LOG_DIR}/.coverage" ./.coverage+ fi+ echo "Running test for retry attempt ${SYSTEM_JOBATTEMPT} with failed tests:"+ run_pytest "$(retry_selection)" "${LOG_DIR}/unittest_test_all_failed_cases_attempt_${SYSTEM_JOBATTEMPT}.log"+ return 0+}\ No newline at end of filediff --git a/.azure-pipelines/scripts/ut/run_ut.sh b/.azure-pipelines/scripts/ut/run_ut.shindex 831562db8..b9324b958 100644--- a/.azure-pipelines/scripts/ut/run_ut.sh+++ b/.azure-pipelines/scripts/ut/run_ut.sh@@ -5,6 +5,7 @@ test_part=${UT_MODE}source /auto-round/.azure-pipelines/scripts/change_color.shsource /auto-round/.azure-pipelines/scripts/ut/detect_changed_tests.sh+source /auto-round/.azure-pipelines/scripts/ut/retry_failed_tests.shLOG_DIR=/auto-round/log_dirmkdir -p "${LOG_DIR}"@@ -21,24 +22,24 @@ function setup_environment() {export TQDM_MININTERVAL=120export HF_HUB_DISABLE_PROGRESS_BARS=1- # install latest gguf for ut test+ echo "Install latest gguf for ut test ..."cd ~ || exit 1git clone -b master --quiet --single-branch https://github.com/ggml-org/llama.cpp.git && cd llama.cpp/gguf-py && uv pip install .- # install unit report dependencies+ echo "Install unit report dependencies ..."uv pip install pytest-covuv pip install -U chardet- uv pip list- # install auto-round for unit tests+ echo "Install auto-round for unit tests ..."cd /auto-round && uv pip install .export LD_LIBRARY_PATH=${HOME}/.venv/lib/:$LD_LIBRARY_PATHexport FORCE_BF16=1export COVERAGE_RCFILE=/auto-round/.azure-pipelines/scripts/ut/coveragerc/cpu.coveragerc- echo "##[endgroup]"-++ echo "List final dependencies ..."uv pip list+ echo "##[endgroup]"}function print_summary() {@@ -61,6 +62,8 @@ function run_pytest() {local ut_log_name=$2echo "##[group]Running ${test_case}..."+ # Record the test targets so a retry can rerun exactly these cases.+ printf '%s\n' ${test_case} > "${ut_log_name%.log}.list"numactl --physcpubind="${NUMA_CPUSET:-0-15}" --membind="${NUMA_NODE:-0}" \pytest -m "not skip_ci" --cov=auto_round --cov-report= --cov-append -vs \--junitxml="${ut_log_name%.log}.xml" ${test_case} 2>&1 | tee ${ut_log_name}@@ -83,6 +86,7 @@ function run_common_group() {function run_common_unit_test() {cd /auto-round/test || exit 1+ run_if_retry && return 0# common test case for cpu/gpu/xpu# Group cases by the first-level folder under unit/common; a single test@@ -100,6 +104,7 @@ function run_common_unit_test() {function run_unit_test() {cd /auto-round/test || exit 1+ run_if_retry && return 0# Split cpu specific test files into 4 parts.# Only fast unit tests run in PR CI; integration (inc/llmc) and e2e suites@@ -146,6 +151,7 @@ function run_inc_unit_test() {echo "##[endgroup]"cd /auto-round/test/integration || exit 1+ run_if_retry && return 0for test_file in ${selected_files}; dolocal test_basename=$(basename ${test_file} .py)@@ -170,6 +176,7 @@ function run_llmc_unit_test() {echo "##[endgroup]"cd /auto-round/test/integration || exit 1+ run_if_retry && return 0for test_file in ${selected_files}; dolocal test_basename=$(basename ${test_file} .py)@@ -180,11 +187,17 @@ function run_llmc_unit_test() {function collect_log() {touch "${SUMMARY_LOG}"+ # collect_result.py also stages only the failed logs for the AI-analysis stage.python /auto-round/.azure-pipelines/scripts/ut/collect_result.py \- --test-type "Unit Tests" --log-pattern "unittest_test_*.log" --log-dir ${LOG_DIR} --summary-log ${SUMMARY_LOG}+ --test-type "Unit Tests" --log-pattern "unittest_test_*.log" --log-dir ${LOG_DIR} \+ --summary-log ${SUMMARY_LOG} --failed-logs-dir "${LOG_DIR}/failed_logs"if [ -f .coverage ]; thencp .coverage "${LOG_DIR}/.coverage.part${test_part}"+ # Keep .coverage in the failure artifact so a retry can accumulate onto it.+ if [ -d "${LOG_DIR}/failed_logs" ]; then+ cp .coverage "${LOG_DIR}/failed_logs/.coverage"+ fifi}diff --git a/.azure-pipelines/scripts/ut/run_ut_xpu.sh b/.azure-pipelines/scripts/ut/run_ut_xpu.shindex 8210a9cf0..d3fa7a297 100644--- a/.azure-pipelines/scripts/ut/run_ut_xpu.sh+++ b/.azure-pipelines/scripts/ut/run_ut_xpu.sh@@ -5,24 +5,28 @@ test_part=${UT_MODE}source /auto-round/.azure-pipelines/scripts/change_color.shsource /auto-round/.azure-pipelines/scripts/ut/detect_changed_tests.sh+source /auto-round/.azure-pipelines/scripts/ut/retry_failed_tests.shTIMEOUT=30SESSION_TIMEOUT=600function setup_environment() {echo "##[group]set up UT env..."+ echo "Install unit report dependencies ..."uv pip install pytest-covuv pip install -U chardet- uv pip list- echo "##[endgroup]"# Keep the GGUF conversion helpers in sync with the model conversion code.# The common GGUF tests exercise MODEL_ARCH entries that are only available# in the current llama.cpp master branch.+ echo "Install latest gguf for ut test ..."cd ~ || exit 1git clone -b master --quiet --single-branch https://github.com/ggml-org/llama.cpp.git \&& cd llama.cpp/gguf-py \&& uv pip install .+ echo "List final dependencies ..."+ uv pip list+ echo "##[endgroup]"git config --global --add safe.directory /auto-roundcd /auto-round/test || exit 1@@ -49,12 +53,15 @@ function run_pytest() {local ut_log_name=$2echo "##[group]Running ${test_case}..."+ # Record the test targets so a retry can rerun exactly these cases.+ printf '%s\n' ${test_case} > "${ut_log_name%.log}.list"numactl --physcpubind="${NUMA_CPUSET:-0-27}" --membind="${NUMA_NODE:-0}" \pytest -m "not skip_ci" --cov=auto_round --cov-report= --cov-append -vs \--junitxml="${ut_log_name%.log}.xml" ${test_case} 2>&1 | tee ${ut_log_name}echo "##[endgroup]"}+function run_common_group() {# Run a group of common test files together in a single pytest invocation.# $1: group name (used for log file), remaining args: test files@@ -70,6 +77,8 @@ function run_common_group() {}function run_common_unit_test() {+ cd /auto-round/test || exit 1+ run_if_retry && return 0# common test case for cpu/gpu/xpu# Group cases by the first-level folder under unit/common; a single test# file placed directly under unit/common (e.g. test_main.py) runs on its own.@@ -85,9 +94,16 @@ function run_common_unit_test() {}function run_unit_test() {+ cd /auto-round/test || exit 1+ run_if_retry && return 0+local xpu_testsxpu_tests=$(filter_changed_tests "test" "$(find ./unit/test_xpu -name "test*.py" | sort)")-+ if [ -z "${xpu_tests}" ]; then+ echo "No changed XPU test file, skip."+ return 0+ fi+for test_file in ${xpu_tests}; dolocal test_basename=$(basename ${test_file} .py)local ut_log_name="${LOG_DIR}/unittest_${test_basename}.log"@@ -96,8 +112,15 @@ function run_unit_test() {}function run_unit_test_ark() {+ cd /auto-round/test || exit 1+ run_if_retry && return 0+local ark_testsark_tests=$(filter_changed_tests "test" "$(find ./unit/test_ark -name "test*.py" | sort)")+ if [ -z "${ark_tests}" ]; then+ echo "No changed ARK test file, skip."+ return 0+ fifor test_file in ${ark_tests}; dolocal test_basename=$(basename ${test_file} .py)@@ -107,6 +130,9 @@ function run_unit_test_ark() {}function run_unit_test_llmc() {+ cd /auto-round/test || exit 1+ run_if_retry && return 0+local llmc_testsllmc_tests=$(filter_changed_tests "test" "$(find ./integration/test_xpu -name "test_llmc_integration.py" | sort)")if [ -z "${llmc_tests}" ]; then@@ -131,39 +157,31 @@ function print_summary() {exit $?}+function check_storage_usage() {+ echo "##[group]check storage usage..."+ df -h+ du -sh /auto-round || true+ du -sh /home/hostuser/.cache/huggingface || true+ du -sh /home/hostuser/.cache/huggingface/hub/* || true+ du -sh /home/hostuser/.venv || true+ echo "##[endgroup]"+}+function collect_log() {touch "${SUMMARY_LOG}"python /auto-round/.azure-pipelines/scripts/ut/collect_result.py \- --test-type "Unit Tests" --log-pattern "unittest_*.log" --log-dir ${LOG_DIR} --summary-log ${SUMMARY_LOG}+ --test-type "Unit Tests" --log-pattern "unittest_*.log" --log-dir ${LOG_DIR} \+ --summary-log ${SUMMARY_LOG} --failed-logs-dir "${LOG_DIR}/failed_logs"+if [ -f .coverage ]; then- # Suffix with the matrix part so the nightly Coverage stage can combine- # the parts without the artifacts overwriting each other.- cp .coverage "${LOG_DIR}/.coverage.${UT_MODE:-base}"- # The tests import auto_round from site-packages, so the raw data holds- # container-only paths. "[paths] source" rewrites those to repo-relative- # ones, but coverage.py applies it during "combine" only -- reporting on- # its own emits /home/hostuser/.venv/... which no agent can resolve, and- # PublishCodeCoverageResults then reports "file does not exist".- # Combine from the repo root so the relative "auto_round" entry matches.- (- cd /auto-round || exit 1- rm -f .coverage- python -m coverage combine --keep "${LOG_DIR}/.coverage.${UT_MODE:-base}"- python -m coverage xml -o "${LOG_DIR}/coverage.xml"- python -m coverage html -d "${LOG_DIR}/htmlcov"- )- else- echo "No coverage data (no test selected), skip coverage report."- echo "##vso[task.setvariable variable=HAS_COVERAGE]false"+ cp .coverage "${LOG_DIR}/.coverage.part${test_part}"+ # Keep .coverage in the failure artifact so a retry can accumulate onto it.+ if [ -d "${LOG_DIR}/failed_logs" ]; then+ cp .coverage "${LOG_DIR}/failed_logs/.coverage"+ fifi}-function print_coverage() {- echo "##[group]overall code coverage..."- [ -f .coverage ] && python -m coverage report || echo "No coverage data."- echo "##[endgroup]"-}-function main() {setup_environmentinit_changed_tests@@ -180,7 +198,7 @@ function main() {echo "invalid name $test_part"ficollect_log- print_coverage+ check_storage_usageprint_summary}diff --git a/.azure-pipelines/template/ut-template.yml b/.azure-pipelines/template/ut-template.ymlindex 29956b645..3c5d7c73a 100644--- a/.azure-pipelines/template/ut-template.yml+++ b/.azure-pipelines/template/ut-template.yml@@ -81,6 +81,43 @@ steps:fidisplayName: "Env Setup"+ # On a "Rerun failed jobs" retry the container is a clean environment, so pull+ # the previous attempt's failed_tests.list to rerun only the failed cases.+ # A cancelled previous attempt never publishes its failed-logs artifact, so+ # probe for it first and skip the download when absent.+ - task: Bash@3+ condition: gt(variables['System.JobAttempt'], 1)+ displayName: "Check previous attempt artifact exists"+ env:+ SYSTEM_ACCESSTOKEN: $(System.AccessToken)+ inputs:+ targetType: "inline"+ script: |+ set -eo pipefail+ prev_attempt=$((SYSTEM_JOBATTEMPT - 1))+ echo "##vso[task.setvariable variable=prevAttempt]${prev_attempt}"+ artifact="${{ parameters.utArtifact }}-${prev_attempt}"+ url="${SYSTEM_COLLECTIONURI}${SYSTEM_TEAMPROJECT}/_apis/build/builds/${BUILD_BUILDID}/artifacts?api-version=7.1"+ resp=$(curl -fsS -H "Authorization: Bearer ${SYSTEM_ACCESSTOKEN}" "${url}" || echo "")+ if echo "${resp}" | grep -q "\"name\":\"${artifact}\""; then+ echo "Found previous artifact ${artifact}."+ echo "##vso[task.setvariable variable=prevArtifactExists]true"+ else+ echo "Previous artifact ${artifact} not found, skip retry download."+ echo "##vso[task.setvariable variable=prevArtifactExists]false"+ fi++ - task: DownloadPipelineArtifact@2+ condition: and(gt(variables['System.JobAttempt'], 1), eq(variables['prevArtifactExists'], 'true'))+ continueOnError: true+ inputs:+ artifact: "${{ parameters.utArtifact }}-$(prevAttempt)"+ patterns: |+ failed_tests.list+ .coverage+ path: ${{ parameters.uploadPath }}+ displayName: "Download previous failed_tests.list and coverage (retry)"+- script: |set -eo pipefail@@ -89,6 +126,7 @@ steps:-e ZE_AFFINITY_MASK=${ZE_AFFINITY_MASK} \-e UT_MODE=${{ parameters.utTestMode }} \-e BUILD_REASON=${BUILD_REASON} \+ -e SYSTEM_JOBATTEMPT=${SYSTEM_JOBATTEMPT} \-e SYSTEM_PULLREQUEST_TARGETBRANCH=${SYSTEM_PULLREQUEST_TARGETBRANCH} \-e FILTER_CHANGED_TESTS=${{ lower(parameters.filterChangedTests) }} \${{ parameters.utContainerName }} \@@ -106,10 +144,10 @@ steps:- task: PublishPipelineArtifact@1condition: failed()inputs:- targetPath: ${{ parameters.uploadPath }}+ targetPath: ${{ parameters.uploadPath }}/failed_logsartifact: "${{ parameters.utArtifact }}-$(System.JobAttempt)"publishLocation: "pipeline"- displayName: "Publish UT logs for debugging"+ displayName: "Publish failed UT logs"- task: Bash@3condition: always()diff --git a/.azure-pipelines/unit-test-cuda.yml b/.azure-pipelines/unit-test-cuda.ymlindex d664f6fe0..95581aa04 100644--- a/.azure-pipelines/unit-test-cuda.yml+++ b/.azure-pipelines/unit-test-cuda.yml@@ -41,12 +41,16 @@ parameters:stages:- ${{ each pair in parameters.matrix }}:- - stage: Start_Pod_${{ pair.key }}- displayName: "Start Pod ${{ pair.value.PART }}"+ # Pod creation, GPU tests and cleanup live in ONE stage as three jobs so a+ # "Rerun failed stage" reruns every job (including Start_Pod_Job) and thus+ # recreates the ephemeral pod. Separate stages would leave the already+ # succeeded Start_Pod stage un-rerun, so the retry would find no agent.+ - stage: GPU_Test_${{ pair.key }}+ displayName: "GPU Test ${{ pair.value.PART }}"dependsOn: []jobs:- job: Start_Pod_Job- displayName: "Launch Pod & Run CUDA Tests"+ displayName: "Launch Pod ${{ pair.value.PART }}"timeoutInMinutes: 60pool:vmImage: "ubuntu-latest"@@ -109,11 +113,8 @@ stages:env:PYTHONUNBUFFERED: '1'- - stage: GPU_Test_${{ pair.key }}- displayName: "GPU Test ${{ pair.value.PART }}"- dependsOn: Start_Pod_${{ pair.key }}- jobs:- job: GpuTests+ dependsOn: Start_Pod_JobtimeoutInMinutes: 60pool:name: "$(POOL_NAME)"@@ -138,6 +139,43 @@ stages:echo "##[endgroup]"displayName: "Install Python Dependencies"+ # On a "Rerun failed jobs" retry the container is a clean environment, so pull+ # the previous attempt's failed_tests.list to rerun only the failed cases.+ # A cancelled previous attempt never publishes its failed-logs artifact, so+ # probe for it first and skip the download when absent.+ - task: Bash@3+ condition: gt(variables['System.JobAttempt'], 1)+ displayName: "Check previous attempt artifact exists"+ env:+ SYSTEM_ACCESSTOKEN: $(System.AccessToken)+ inputs:+ targetType: "inline"+ script: |+ set -eo pipefail+ prev_attempt=$((SYSTEM_JOBATTEMPT - 1))+ echo "##vso[task.setvariable variable=prevAttempt]${prev_attempt}"+ artifact="ut-${{ pair.value.PART }}-${prev_attempt}"+ url="${SYSTEM_COLLECTIONURI}${SYSTEM_TEAMPROJECT}/_apis/build/builds/${BUILD_BUILDID}/artifacts?api-version=7.1"+ resp=$(curl -fsS -H "Authorization: Bearer ${SYSTEM_ACCESSTOKEN}" "${url}" || echo "")+ if echo "${resp}" | grep -q "\"name\":\"${artifact}\""; then+ echo "Found previous artifact ${artifact}."+ echo "##vso[task.setvariable variable=prevArtifactExists]true"+ else+ echo "Previous artifact ${artifact} not found, skip retry download."+ echo "##vso[task.setvariable variable=prevArtifactExists]false"+ fi++ - task: DownloadPipelineArtifact@2+ condition: and(gt(variables['System.JobAttempt'], 1), eq(variables['prevArtifactExists'], 'true'))+ continueOnError: true+ inputs:+ artifact: "ut-${{ pair.value.PART }}-$(prevAttempt)"+ patterns: |+ failed_tests.list+ .coverage+ path: $(Build.SourcesDirectory)/log_dir+ displayName: "Download previous failed_tests.list and coverage (retry)"+- script: |export UV_NO_CACHE=0export FILTER_CHANGED_TESTS=true@@ -154,20 +192,28 @@ stages:publishLocation: "pipeline"displayName: "Publish UT coverage artifact"+ # Publish only the failed logs (and seeded .coverage) so the next+ # retry attempt can download and rerun exactly those cases.+ - task: PublishPipelineArtifact@1+ condition: failed()+ inputs:+ targetPath: $(Build.SourcesDirectory)/log_dir/failed_logs+ artifact: "ut-${{ pair.value.PART }}-$(System.JobAttempt)"+ publishLocation: "pipeline"+ displayName: "Publish failed UT logs"+- script: |cd .azure-pipelines/scripts/cuda_unit_testuv run monitor_gpu.py stopcondition: always()displayName: "GPU Monitor"- - stage: Cleanup_Verification_${{ pair.key }}- displayName: "Verify Pod Cleanup ${{ pair.value.PART }}"- dependsOn:- - Start_Pod_${{ pair.key }}- - GPU_Test_${{ pair.key }}- condition: always()- jobs:- job: Verify_Cleanup+ displayName: "Verify Pod Cleanup ${{ pair.value.PART }}"+ dependsOn:+ - Start_Pod_Job+ - GpuTests+ condition: always()pool:vmImage: "ubuntu-latest"steps:@@ -223,7 +269,6 @@ stages:dependsOn:- ${{ each pair in parameters.matrix }}:- GPU_Test_${{ pair.key }}- - Cleanup_Verification_${{ pair.key }}variables:UPLOAD_PATH: $(Build.SourcesDirectory)/log_dirDOWNLOAD_PATH: $(Build.SourcesDirectory)/log_dirdiff --git a/.azure-pipelines/unit-test-xpu.yml b/.azure-pipelines/unit-test-xpu.ymlindex 2bdc110ff..0ec992d90 100644--- a/.azure-pipelines/unit-test-xpu.yml+++ b/.azure-pipelines/unit-test-xpu.yml@@ -118,10 +118,6 @@ stages:publishLocation: "pipeline"- task: PublishCodeCoverageResults@2- condition: ne(variables['HAS_COVERAGE'], 'false')inputs:- summaryFileLocation: $(UPLOAD_PATH)/coverage_PR/coverage.xml- # coverage.xml is produced inside the container, where the checkout- # lives at /auto-round, so its <sources> root does not exist on the- # agent. Point the task at the agent-side checkout instead.- pathToSources: $(Build.SourcesDirectory)+ summaryFileLocation: $(Build.SourcesDirectory)/log_dir/coverage_PR/coverage.xml+diff --git a/test/pytest.ini b/test/pytest.iniindex 6e41873ea..fc06b61a9 100644--- a/test/pytest.ini+++ b/test/pytest.ini@@ -3,4 +3,4 @@ testpaths = unitpython_files = test_*.pypython_classes = Test*python_functions = test_*-addopts = -vs --tb=short --ignore=test/unit/test_xpu --durations=0 --durations-min=1 --disable-warnings+addopts = -vs --tb=short --durations=0 --durations-min=1 --disable-warnings --show-capture=no