Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Enhance test logging and retry mechanism for unit tests (#2274)

chen, suyueb48c9db6744e
diff --git a/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh b/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh
index cf05383b6..dd1b83167 100644
--- a/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh
+++ b/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh
@@ -26,6 +26,8 @@ source ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/detect_changed_test
LOG_DIR="${BUILD_SOURCESDIRECTORY}/log_dir"
mkdir -p "${LOG_DIR}"
+
+source ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/retry_failed_tests.sh
SUMMARY_LOG="${LOG_DIR}/results_summary.log"
# print_summary reads this file unconditionally; a matrix part that selects no
# test never writes it, so make sure it always exists.
@@ -69,9 +71,11 @@ function setup_basic_test_env() {
uv pip uninstall torch torchvision
uv pip install torch==2.13.0 torchvision torchao --index-url https://download.pytorch.org/whl/cu130
uv pip install .
- echo "##[endgroup]"
+ echo "List dependencies ..."
uv pip list
+ echo "##[endgroup]"
+
cd "${BUILD_SOURCESDIRECTORY}/test" || exit 1
}
@@ -80,6 +84,8 @@ function run_pytest() {
local ut_log_name=$2
echo "##[group]Running ${test_case}..."
+ # Record the test targets so a retry can rerun exactly these cases.
+ printf '%s\n' ${test_case} > "${ut_log_name%.log}.list"
pytest -m "not skip_ci" --cov=auto_round --cov-report= --cov-append -vs \
--junitxml="${ut_log_name%.log}.xml" ${test_case} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
@@ -100,6 +106,8 @@ function run_common_group() {
}
function run_common_unit_test() {
+ run_if_retry && return 0
+
# common test case for cpu/gpu/xpu
# Group cases by the first-level folder under unit/common; a single test
# file placed directly under unit/common (e.g. test_main.py) runs on its own.
@@ -116,6 +124,8 @@ function run_common_unit_test() {
function run_unit_test() {
+ run_if_retry && return 0
+
# run ci cuda ut scope
find ./unit/test_cuda -type f -name "test_*.py" | grep -Ev "vlms|llmc|sglang|vllm|multiple_card" | sort > all_tests.txt
total_lines=$(wc -l < all_tests.txt)
@@ -221,11 +231,17 @@ function run_unit_test_vllm() {
function collect_log() {
touch "${SUMMARY_LOG}"
+ # collect_result.py also stages only the failed logs so a retry can rerun them.
python ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/collect_result.py \
- --test-type "Unit Tests" --log-pattern "unittest_cuda_*.log" --log-dir ${LOG_DIR} --summary-log ${SUMMARY_LOG}
+ --test-type "Unit Tests" --log-pattern "unittest_*.log" --log-dir ${LOG_DIR} \
+ --summary-log ${SUMMARY_LOG} --failed-logs-dir "${LOG_DIR}/failed_logs"
if [ -f .coverage ]; then
cp .coverage "${LOG_DIR}/.coverage.part${test_part}"
+ # Keep .coverage in the failure artifact so a retry can accumulate onto it.
+ if [ -d "${LOG_DIR}/failed_logs" ]; then
+ cp .coverage "${LOG_DIR}/failed_logs/.coverage"
+ fi
fi
}
diff --git a/.azure-pipelines/scripts/ut/collect_result.py b/.azure-pipelines/scripts/ut/collect_result.py
index 58a420a19..0b23ad9e2 100644
--- a/.azure-pipelines/scripts/ut/collect_result.py
+++ b/.azure-pipelines/scripts/ut/collect_result.py
@@ -13,7 +13,10 @@ as FAILED.
"""
import argparse
+import os
+import shutil
import sys
+import tempfile
import xml.etree.ElementTree as ET
from dataclasses import dataclass
from enum import Enum, auto
@@ -46,6 +49,7 @@ class TestResult:
status: TestStatus
duration: str
counts: TestCounts
+ log_path: Path | None = None
class XmlAnalyzer:
@@ -89,6 +93,7 @@ class XmlAnalyzer:
status=self._determine_status(counts, xml_file),
duration=duration,
counts=counts if counts is not None else TestCounts(),
+ log_path=log_file,
)
def _extract_test_name(self, path: Path) -> str:
@@ -229,12 +234,66 @@ class ReportGenerator:
)
+def failed_log_paths(results: list[TestResult]) -> list[Path]:
+ """Return the log paths of the failed tests."""
+ return [r.log_path for r in results if r.status == TestStatus.FAILED and r.log_path and r.log_path.is_file()]
+
+
+def stage_failed_logs(log_paths: list[Path], failed_dir: Path) -> None:
+ """Copy the failed test logs into ``failed_dir`` and assemble failed_tests.list.
+
+ ``failed_tests.list`` records the test targets of every failed log (read from
+ the sibling ``.list`` files) so a clean-environment retry can rerun exactly
+ those cases. Centralized here so every hardware UT runner behaves the same.
+
+ ``failed_tests.list`` is written atomically (temp file + rename) so an
+ interruption cannot leave a partial list that would shrink the retry scope.
+ """
+ failed_dir.mkdir(parents=True, exist_ok=True)
+ failed_cases: list[str] = []
+ seen: set[str] = set()
+ for log_path in log_paths:
+ shutil.copy2(log_path, failed_dir / log_path.name)
+ list_file = log_path.with_suffix(".list")
+ if list_file.is_file():
+ for case in list_file.read_text(encoding="utf-8").split():
+ if case not in seen:
+ seen.add(case)
+ failed_cases.append(case)
+ _atomic_write_text(
+ failed_dir / "failed_tests.list",
+ "\n".join(failed_cases) + ("\n" if failed_cases else ""),
+ )
+ print(
+ f"Staged {len(log_paths)} failed log(s) and {len(failed_cases)} case(s) " f"into: {failed_dir.absolute()}",
+ file=sys.stderr,
+ )
+
+
+def _atomic_write_text(target: Path, text: str) -> None:
+ """Write ``text`` to ``target`` atomically via a temp file in the same dir."""
+ fd, tmp_name = tempfile.mkstemp(dir=str(target.parent), prefix=f".{target.name}.", suffix=".tmp")
+ try:
+ with os.fdopen(fd, "w", encoding="utf-8") as f:
+ f.write(text)
+ f.flush()
+ os.fsync(f.fileno())
+ os.replace(tmp_name, target)
+ except BaseException:
+ try:
+ os.unlink(tmp_name)
+ except OSError:
+ pass
+ raise
+
+
def main():
parser = argparse.ArgumentParser(description="Analyze JUnit XML reports and generate summary")
parser.add_argument("--test-type", required=True, help="Type of tests")
parser.add_argument("--log-dir", required=True, type=Path, help="Directory with logs/reports")
parser.add_argument("--summary-log", required=True, type=Path, help="Output file")
parser.add_argument("--log-pattern", default="*.log", help="Glob pattern for log files")
+ parser.add_argument("--failed-logs-dir", type=Path, help="Optional dir to stage only the failed test logs into")
args = parser.parse_args()
@@ -262,6 +321,9 @@ def main():
f"{stats['failed']} failed, {stats['skipped']} skipped"
)
+ if args.failed_logs_dir is not None and failed > 0:
+ stage_failed_logs(failed_log_paths(results), args.failed_logs_dir)
+
except Exception as e:
print(f"Error: {e}", file=sys.stderr)
import traceback
diff --git a/.azure-pipelines/scripts/ut/detect_changed_tests.sh b/.azure-pipelines/scripts/ut/detect_changed_tests.sh
index 8753d4dcb..9c2824eaa 100644
--- a/.azure-pipelines/scripts/ut/detect_changed_tests.sh
+++ b/.azure-pipelines/scripts/ut/detect_changed_tests.sh
@@ -87,8 +87,8 @@ detect_changed_tests() {
<(printf '%s\n' "${test_changed}" | sort -u))"
if [ -n "${non_test}" ]; then
echo "##[group]non-test files changed, run full suite:" >&2
- printf ' %s\n' ${non_test} >&2
- echo "##[endgroup]"
+ printf ' %s\n' "${non_test}" >&2
+ echo "##[endgroup]" >&2
return 1
fi
diff --git a/.azure-pipelines/scripts/ut/retry_failed_tests.sh b/.azure-pipelines/scripts/ut/retry_failed_tests.sh
new file mode 100644
index 000000000..9b0e3386f
--- /dev/null
+++ b/.azure-pipelines/scripts/ut/retry_failed_tests.sh
@@ -0,0 +1,24 @@
+# A retry is any job attempt after the first; failed_tests.list is downloaded
+# from the previous attempt before this script runs (clean environment).
+
+SYSTEM_JOBATTEMPT=${SYSTEM_JOBATTEMPT:-1}
+LOG_DIR=${LOG_DIR:-/auto-round/log_dir}
+
+function is_retry() {
+ [ "${SYSTEM_JOBATTEMPT:-1}" -gt 1 ] && [ -s "${LOG_DIR}/failed_tests.list" ]
+}
+
+function retry_selection() {
+ is_retry && sort -u "${LOG_DIR}/failed_tests.list"
+}
+
+function run_if_retry() {
+ is_retry || return 1
+ # Seed coverage from the previous attempt so --cov-append accumulates.
+ if [ -f "${LOG_DIR}/.coverage" ]; then
+ cp "${LOG_DIR}/.coverage" ./.coverage
+ fi
+ echo "Running test for retry attempt ${SYSTEM_JOBATTEMPT} with failed tests:"
+ run_pytest "$(retry_selection)" "${LOG_DIR}/unittest_test_all_failed_cases_attempt_${SYSTEM_JOBATTEMPT}.log"
+ return 0
+}
\ No newline at end of file
diff --git a/.azure-pipelines/scripts/ut/run_ut.sh b/.azure-pipelines/scripts/ut/run_ut.sh
index 831562db8..b9324b958 100644
--- a/.azure-pipelines/scripts/ut/run_ut.sh
+++ b/.azure-pipelines/scripts/ut/run_ut.sh
@@ -5,6 +5,7 @@ test_part=${UT_MODE}
source /auto-round/.azure-pipelines/scripts/change_color.sh
source /auto-round/.azure-pipelines/scripts/ut/detect_changed_tests.sh
+source /auto-round/.azure-pipelines/scripts/ut/retry_failed_tests.sh
LOG_DIR=/auto-round/log_dir
mkdir -p "${LOG_DIR}"
@@ -21,24 +22,24 @@ function setup_environment() {
export TQDM_MININTERVAL=120
export HF_HUB_DISABLE_PROGRESS_BARS=1
- # install latest gguf for ut test
+ echo "Install latest gguf for ut test ..."
cd ~ || exit 1
git clone -b master --quiet --single-branch https://github.com/ggml-org/llama.cpp.git && cd llama.cpp/gguf-py && uv pip install .
- # install unit report dependencies
+ echo "Install unit report dependencies ..."
uv pip install pytest-cov
uv pip install -U chardet
- uv pip list
- # install auto-round for unit tests
+ echo "Install auto-round for unit tests ..."
cd /auto-round && uv pip install .
export LD_LIBRARY_PATH=${HOME}/.venv/lib/:$LD_LIBRARY_PATH
export FORCE_BF16=1
export COVERAGE_RCFILE=/auto-round/.azure-pipelines/scripts/ut/coveragerc/cpu.coveragerc
- echo "##[endgroup]"
-
+
+ echo "List final dependencies ..."
uv pip list
+ echo "##[endgroup]"
}
function print_summary() {
@@ -61,6 +62,8 @@ function run_pytest() {
local ut_log_name=$2
echo "##[group]Running ${test_case}..."
+ # Record the test targets so a retry can rerun exactly these cases.
+ printf '%s\n' ${test_case} > "${ut_log_name%.log}.list"
numactl --physcpubind="${NUMA_CPUSET:-0-15}" --membind="${NUMA_NODE:-0}" \
pytest -m "not skip_ci" --cov=auto_round --cov-report= --cov-append -vs \
--junitxml="${ut_log_name%.log}.xml" ${test_case} 2>&1 | tee ${ut_log_name}
@@ -83,6 +86,7 @@ function run_common_group() {
function run_common_unit_test() {
cd /auto-round/test || exit 1
+ run_if_retry && return 0
# common test case for cpu/gpu/xpu
# Group cases by the first-level folder under unit/common; a single test
@@ -100,6 +104,7 @@ function run_common_unit_test() {
function run_unit_test() {
cd /auto-round/test || exit 1
+ run_if_retry && return 0
# Split cpu specific test files into 4 parts.
# Only fast unit tests run in PR CI; integration (inc/llmc) and e2e suites
@@ -146,6 +151,7 @@ function run_inc_unit_test() {
echo "##[endgroup]"
cd /auto-round/test/integration || exit 1
+ run_if_retry && return 0
for test_file in ${selected_files}; do
local test_basename=$(basename ${test_file} .py)
@@ -170,6 +176,7 @@ function run_llmc_unit_test() {
echo "##[endgroup]"
cd /auto-round/test/integration || exit 1
+ run_if_retry && return 0
for test_file in ${selected_files}; do
local test_basename=$(basename ${test_file} .py)
@@ -180,11 +187,17 @@ function run_llmc_unit_test() {
function collect_log() {
touch "${SUMMARY_LOG}"
+ # collect_result.py also stages only the failed logs for the AI-analysis stage.
python /auto-round/.azure-pipelines/scripts/ut/collect_result.py \
- --test-type "Unit Tests" --log-pattern "unittest_test_*.log" --log-dir ${LOG_DIR} --summary-log ${SUMMARY_LOG}
+ --test-type "Unit Tests" --log-pattern "unittest_test_*.log" --log-dir ${LOG_DIR} \
+ --summary-log ${SUMMARY_LOG} --failed-logs-dir "${LOG_DIR}/failed_logs"
if [ -f .coverage ]; then
cp .coverage "${LOG_DIR}/.coverage.part${test_part}"
+ # Keep .coverage in the failure artifact so a retry can accumulate onto it.
+ if [ -d "${LOG_DIR}/failed_logs" ]; then
+ cp .coverage "${LOG_DIR}/failed_logs/.coverage"
+ fi
fi
}
diff --git a/.azure-pipelines/scripts/ut/run_ut_xpu.sh b/.azure-pipelines/scripts/ut/run_ut_xpu.sh
index 8210a9cf0..d3fa7a297 100644
--- a/.azure-pipelines/scripts/ut/run_ut_xpu.sh
+++ b/.azure-pipelines/scripts/ut/run_ut_xpu.sh
@@ -5,24 +5,28 @@ test_part=${UT_MODE}
source /auto-round/.azure-pipelines/scripts/change_color.sh
source /auto-round/.azure-pipelines/scripts/ut/detect_changed_tests.sh
+source /auto-round/.azure-pipelines/scripts/ut/retry_failed_tests.sh
TIMEOUT=30
SESSION_TIMEOUT=600
function setup_environment() {
echo "##[group]set up UT env..."
+ echo "Install unit report dependencies ..."
uv pip install pytest-cov
uv pip install -U chardet
- uv pip list
- echo "##[endgroup]"
# Keep the GGUF conversion helpers in sync with the model conversion code.
# The common GGUF tests exercise MODEL_ARCH entries that are only available
# in the current llama.cpp master branch.
+ echo "Install latest gguf for ut test ..."
cd ~ || exit 1
git clone -b master --quiet --single-branch https://github.com/ggml-org/llama.cpp.git \
&& cd llama.cpp/gguf-py \
&& uv pip install .
+ echo "List final dependencies ..."
+ uv pip list
+ echo "##[endgroup]"
git config --global --add safe.directory /auto-round
cd /auto-round/test || exit 1
@@ -49,12 +53,15 @@ function run_pytest() {
local ut_log_name=$2
echo "##[group]Running ${test_case}..."
+ # Record the test targets so a retry can rerun exactly these cases.
+ printf '%s\n' ${test_case} > "${ut_log_name%.log}.list"
numactl --physcpubind="${NUMA_CPUSET:-0-27}" --membind="${NUMA_NODE:-0}" \
pytest -m "not skip_ci" --cov=auto_round --cov-report= --cov-append -vs \
--junitxml="${ut_log_name%.log}.xml" ${test_case} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
}
+
function run_common_group() {
# Run a group of common test files together in a single pytest invocation.
# $1: group name (used for log file), remaining args: test files
@@ -70,6 +77,8 @@ function run_common_group() {
}
function run_common_unit_test() {
+ cd /auto-round/test || exit 1
+ run_if_retry && return 0
# common test case for cpu/gpu/xpu
# Group cases by the first-level folder under unit/common; a single test
# file placed directly under unit/common (e.g. test_main.py) runs on its own.
@@ -85,9 +94,16 @@ function run_common_unit_test() {
}
function run_unit_test() {
+ cd /auto-round/test || exit 1
+ run_if_retry && return 0
+
local xpu_tests
xpu_tests=$(filter_changed_tests "test" "$(find ./unit/test_xpu -name "test*.py" | sort)")
-
+ if [ -z "${xpu_tests}" ]; then
+ echo "No changed XPU test file, skip."
+ return 0
+ fi
+
for test_file in ${xpu_tests}; do
local test_basename=$(basename ${test_file} .py)
local ut_log_name="${LOG_DIR}/unittest_${test_basename}.log"
@@ -96,8 +112,15 @@ function run_unit_test() {
}
function run_unit_test_ark() {
+ cd /auto-round/test || exit 1
+ run_if_retry && return 0
+
local ark_tests
ark_tests=$(filter_changed_tests "test" "$(find ./unit/test_ark -name "test*.py" | sort)")
+ if [ -z "${ark_tests}" ]; then
+ echo "No changed ARK test file, skip."
+ return 0
+ fi
for test_file in ${ark_tests}; do
local test_basename=$(basename ${test_file} .py)
@@ -107,6 +130,9 @@ function run_unit_test_ark() {
}
function run_unit_test_llmc() {
+ cd /auto-round/test || exit 1
+ run_if_retry && return 0
+
local llmc_tests
llmc_tests=$(filter_changed_tests "test" "$(find ./integration/test_xpu -name "test_llmc_integration.py" | sort)")
if [ -z "${llmc_tests}" ]; then
@@ -131,39 +157,31 @@ function print_summary() {
exit $?
}
+function check_storage_usage() {
+ echo "##[group]check storage usage..."
+ df -h
+ du -sh /auto-round || true
+ du -sh /home/hostuser/.cache/huggingface || true
+ du -sh /home/hostuser/.cache/huggingface/hub/* || true
+ du -sh /home/hostuser/.venv || true
+ echo "##[endgroup]"
+}
+
function collect_log() {
touch "${SUMMARY_LOG}"
python /auto-round/.azure-pipelines/scripts/ut/collect_result.py \
- --test-type "Unit Tests" --log-pattern "unittest_*.log" --log-dir ${LOG_DIR} --summary-log ${SUMMARY_LOG}
+ --test-type "Unit Tests" --log-pattern "unittest_*.log" --log-dir ${LOG_DIR} \
+ --summary-log ${SUMMARY_LOG} --failed-logs-dir "${LOG_DIR}/failed_logs"
+
if [ -f .coverage ]; then
- # Suffix with the matrix part so the nightly Coverage stage can combine
- # the parts without the artifacts overwriting each other.
- cp .coverage "${LOG_DIR}/.coverage.${UT_MODE:-base}"
- # The tests import auto_round from site-packages, so the raw data holds
- # container-only paths. "[paths] source" rewrites those to repo-relative
- # ones, but coverage.py applies it during "combine" only -- reporting on
- # its own emits /home/hostuser/.venv/... which no agent can resolve, and
- # PublishCodeCoverageResults then reports "file does not exist".
- # Combine from the repo root so the relative "auto_round" entry matches.
- (
- cd /auto-round || exit 1
- rm -f .coverage
- python -m coverage combine --keep "${LOG_DIR}/.coverage.${UT_MODE:-base}"
- python -m coverage xml -o "${LOG_DIR}/coverage.xml"
- python -m coverage html -d "${LOG_DIR}/htmlcov"
- )
- else
- echo "No coverage data (no test selected), skip coverage report."
- echo "##vso[task.setvariable variable=HAS_COVERAGE]false"
+ cp .coverage "${LOG_DIR}/.coverage.part${test_part}"
+ # Keep .coverage in the failure artifact so a retry can accumulate onto it.
+ if [ -d "${LOG_DIR}/failed_logs" ]; then
+ cp .coverage "${LOG_DIR}/failed_logs/.coverage"
+ fi
fi
}
-function print_coverage() {
- echo "##[group]overall code coverage..."
- [ -f .coverage ] && python -m coverage report || echo "No coverage data."
- echo "##[endgroup]"
-}
-
function main() {
setup_environment
init_changed_tests
@@ -180,7 +198,7 @@ function main() {
echo "invalid name $test_part"
fi
collect_log
- print_coverage
+ check_storage_usage
print_summary
}
diff --git a/.azure-pipelines/template/ut-template.yml b/.azure-pipelines/template/ut-template.yml
index 29956b645..3c5d7c73a 100644
--- a/.azure-pipelines/template/ut-template.yml
+++ b/.azure-pipelines/template/ut-template.yml
@@ -81,6 +81,43 @@ steps:
fi
displayName: "Env Setup"
+ # On a "Rerun failed jobs" retry the container is a clean environment, so pull
+ # the previous attempt's failed_tests.list to rerun only the failed cases.
+ # A cancelled previous attempt never publishes its failed-logs artifact, so
+ # probe for it first and skip the download when absent.
+ - task: Bash@3
+ condition: gt(variables['System.JobAttempt'], 1)
+ displayName: "Check previous attempt artifact exists"
+ env:
+ SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+ inputs:
+ targetType: "inline"
+ script: |
+ set -eo pipefail
+ prev_attempt=$((SYSTEM_JOBATTEMPT - 1))
+ echo "##vso[task.setvariable variable=prevAttempt]${prev_attempt}"
+ artifact="${{ parameters.utArtifact }}-${prev_attempt}"
+ url="${SYSTEM_COLLECTIONURI}${SYSTEM_TEAMPROJECT}/_apis/build/builds/${BUILD_BUILDID}/artifacts?api-version=7.1"
+ resp=$(curl -fsS -H "Authorization: Bearer ${SYSTEM_ACCESSTOKEN}" "${url}" || echo "")
+ if echo "${resp}" | grep -q "\"name\":\"${artifact}\""; then
+ echo "Found previous artifact ${artifact}."
+ echo "##vso[task.setvariable variable=prevArtifactExists]true"
+ else
+ echo "Previous artifact ${artifact} not found, skip retry download."
+ echo "##vso[task.setvariable variable=prevArtifactExists]false"
+ fi
+
+ - task: DownloadPipelineArtifact@2
+ condition: and(gt(variables['System.JobAttempt'], 1), eq(variables['prevArtifactExists'], 'true'))
+ continueOnError: true
+ inputs:
+ artifact: "${{ parameters.utArtifact }}-$(prevAttempt)"
+ patterns: |
+ failed_tests.list
+ .coverage
+ path: ${{ parameters.uploadPath }}
+ displayName: "Download previous failed_tests.list and coverage (retry)"
+
- script: |
set -eo pipefail
@@ -89,6 +126,7 @@ steps:
-e ZE_AFFINITY_MASK=${ZE_AFFINITY_MASK} \
-e UT_MODE=${{ parameters.utTestMode }} \
-e BUILD_REASON=${BUILD_REASON} \
+ -e SYSTEM_JOBATTEMPT=${SYSTEM_JOBATTEMPT} \
-e SYSTEM_PULLREQUEST_TARGETBRANCH=${SYSTEM_PULLREQUEST_TARGETBRANCH} \
-e FILTER_CHANGED_TESTS=${{ lower(parameters.filterChangedTests) }} \
${{ parameters.utContainerName }} \
@@ -106,10 +144,10 @@ steps:
- task: PublishPipelineArtifact@1
condition: failed()
inputs:
- targetPath: ${{ parameters.uploadPath }}
+ targetPath: ${{ parameters.uploadPath }}/failed_logs
artifact: "${{ parameters.utArtifact }}-$(System.JobAttempt)"
publishLocation: "pipeline"
- displayName: "Publish UT logs for debugging"
+ displayName: "Publish failed UT logs"
- task: Bash@3
condition: always()
diff --git a/.azure-pipelines/unit-test-cuda.yml b/.azure-pipelines/unit-test-cuda.yml
index d664f6fe0..95581aa04 100644
--- a/.azure-pipelines/unit-test-cuda.yml
+++ b/.azure-pipelines/unit-test-cuda.yml
@@ -41,12 +41,16 @@ parameters:
stages:
- ${{ each pair in parameters.matrix }}:
- - stage: Start_Pod_${{ pair.key }}
- displayName: "Start Pod ${{ pair.value.PART }}"
+ # Pod creation, GPU tests and cleanup live in ONE stage as three jobs so a
+ # "Rerun failed stage" reruns every job (including Start_Pod_Job) and thus
+ # recreates the ephemeral pod. Separate stages would leave the already
+ # succeeded Start_Pod stage un-rerun, so the retry would find no agent.
+ - stage: GPU_Test_${{ pair.key }}
+ displayName: "GPU Test ${{ pair.value.PART }}"
dependsOn: []
jobs:
- job: Start_Pod_Job
- displayName: "Launch Pod & Run CUDA Tests"
+ displayName: "Launch Pod ${{ pair.value.PART }}"
timeoutInMinutes: 60
pool:
vmImage: "ubuntu-latest"
@@ -109,11 +113,8 @@ stages:
env:
PYTHONUNBUFFERED: '1'
- - stage: GPU_Test_${{ pair.key }}
- displayName: "GPU Test ${{ pair.value.PART }}"
- dependsOn: Start_Pod_${{ pair.key }}
- jobs:
- job: GpuTests
+ dependsOn: Start_Pod_Job
timeoutInMinutes: 60
pool:
name: "$(POOL_NAME)"
@@ -138,6 +139,43 @@ stages:
echo "##[endgroup]"
displayName: "Install Python Dependencies"
+ # On a "Rerun failed jobs" retry the container is a clean environment, so pull
+ # the previous attempt's failed_tests.list to rerun only the failed cases.
+ # A cancelled previous attempt never publishes its failed-logs artifact, so
+ # probe for it first and skip the download when absent.
+ - task: Bash@3
+ condition: gt(variables['System.JobAttempt'], 1)
+ displayName: "Check previous attempt artifact exists"
+ env:
+ SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+ inputs:
+ targetType: "inline"
+ script: |
+ set -eo pipefail
+ prev_attempt=$((SYSTEM_JOBATTEMPT - 1))
+ echo "##vso[task.setvariable variable=prevAttempt]${prev_attempt}"
+ artifact="ut-${{ pair.value.PART }}-${prev_attempt}"
+ url="${SYSTEM_COLLECTIONURI}${SYSTEM_TEAMPROJECT}/_apis/build/builds/${BUILD_BUILDID}/artifacts?api-version=7.1"
+ resp=$(curl -fsS -H "Authorization: Bearer ${SYSTEM_ACCESSTOKEN}" "${url}" || echo "")
+ if echo "${resp}" | grep -q "\"name\":\"${artifact}\""; then
+ echo "Found previous artifact ${artifact}."
+ echo "##vso[task.setvariable variable=prevArtifactExists]true"
+ else
+ echo "Previous artifact ${artifact} not found, skip retry download."
+ echo "##vso[task.setvariable variable=prevArtifactExists]false"
+ fi
+
+ - task: DownloadPipelineArtifact@2
+ condition: and(gt(variables['System.JobAttempt'], 1), eq(variables['prevArtifactExists'], 'true'))
+ continueOnError: true
+ inputs:
+ artifact: "ut-${{ pair.value.PART }}-$(prevAttempt)"
+ patterns: |
+ failed_tests.list
+ .coverage
+ path: $(Build.SourcesDirectory)/log_dir
+ displayName: "Download previous failed_tests.list and coverage (retry)"
+
- script: |
export UV_NO_CACHE=0
export FILTER_CHANGED_TESTS=true
@@ -154,20 +192,28 @@ stages:
publishLocation: "pipeline"
displayName: "Publish UT coverage artifact"
+ # Publish only the failed logs (and seeded .coverage) so the next
+ # retry attempt can download and rerun exactly those cases.
+ - task: PublishPipelineArtifact@1
+ condition: failed()
+ inputs:
+ targetPath: $(Build.SourcesDirectory)/log_dir/failed_logs
+ artifact: "ut-${{ pair.value.PART }}-$(System.JobAttempt)"
+ publishLocation: "pipeline"
+ displayName: "Publish failed UT logs"
+
- script: |
cd .azure-pipelines/scripts/cuda_unit_test
uv run monitor_gpu.py stop
condition: always()
displayName: "GPU Monitor"
- - stage: Cleanup_Verification_${{ pair.key }}
- displayName: "Verify Pod Cleanup ${{ pair.value.PART }}"
- dependsOn:
- - Start_Pod_${{ pair.key }}
- - GPU_Test_${{ pair.key }}
- condition: always()
- jobs:
- job: Verify_Cleanup
+ displayName: "Verify Pod Cleanup ${{ pair.value.PART }}"
+ dependsOn:
+ - Start_Pod_Job
+ - GpuTests
+ condition: always()
pool:
vmImage: "ubuntu-latest"
steps:
@@ -223,7 +269,6 @@ stages:
dependsOn:
- ${{ each pair in parameters.matrix }}:
- GPU_Test_${{ pair.key }}
- - Cleanup_Verification_${{ pair.key }}
variables:
UPLOAD_PATH: $(Build.SourcesDirectory)/log_dir
DOWNLOAD_PATH: $(Build.SourcesDirectory)/log_dir
diff --git a/.azure-pipelines/unit-test-xpu.yml b/.azure-pipelines/unit-test-xpu.yml
index 2bdc110ff..0ec992d90 100644
--- a/.azure-pipelines/unit-test-xpu.yml
+++ b/.azure-pipelines/unit-test-xpu.yml
@@ -118,10 +118,6 @@ stages:
publishLocation: "pipeline"
- task: PublishCodeCoverageResults@2
- condition: ne(variables['HAS_COVERAGE'], 'false')
inputs:
- summaryFileLocation: $(UPLOAD_PATH)/coverage_PR/coverage.xml
- # coverage.xml is produced inside the container, where the checkout
- # lives at /auto-round, so its <sources> root does not exist on the
- # agent. Point the task at the agent-side checkout instead.
- pathToSources: $(Build.SourcesDirectory)
+ summaryFileLocation: $(Build.SourcesDirectory)/log_dir/coverage_PR/coverage.xml
+
diff --git a/test/pytest.ini b/test/pytest.ini
index 6e41873ea..fc06b61a9 100644
--- a/test/pytest.ini
+++ b/test/pytest.ini
@@ -3,4 +3,4 @@ testpaths = unit
python_files = test_*.py
python_classes = Test*
python_functions = test_*
-addopts = -vs --tb=short --ignore=test/unit/test_xpu --durations=0 --durations-min=1 --disable-warnings
+addopts = -vs --tb=short --durations=0 --durations-min=1 --disable-warnings --show-capture=no