PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Add CUDA CI coverage (#2074)
Sun, Xuehao6ff414b15728
diff --git a/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh b/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.shindex 27ae8e2bd..68b85dffa 100644--- a/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh+++ b/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh@@ -20,7 +20,7 @@ donesource ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/change_color.sh-LOG_DIR="${BUILD_SOURCESDIRECTORY}/ut_log_dir"+LOG_DIR="${BUILD_SOURCESDIRECTORY}/log_dir"mkdir -p "${LOG_DIR}"SUMMARY_LOG="${LOG_DIR}/results_summary.log"@@ -51,10 +51,10 @@ function print_summary() {function check_storage_usage() {echo "##[group]check storage usage..."df -h- du -sh "${BUILD_SOURCESDIRECTORY}"- du -sh /root/.cache/huggingface- du -sh /root/.cache/huggingface/hub/*- du -sh /root/.venv+ du -sh "${BUILD_SOURCESDIRECTORY}" || true+ du -sh /root/.cache/huggingface || true+ du -sh /root/.cache/huggingface/hub/* || true+ du -sh /root/.venv || trueecho "##[endgroup]"}@@ -68,6 +68,7 @@ function run_unit_test() {uv pip install -r test/test_cuda/requirements.txtuv pip install -r test/test_cuda/requirements_diffusion.txtuv pip install -U transformers chardet+ uv pip install -U pytest-covuv pip install kernels==0.15.2 # For sm120: https://github.com/huggingface/transformers/blob/v5.13.1/setup.py#L93uv pip uninstall torch torchvisionuv pip install torch==2.13.0 torchvision torchao --index-url https://download.pytorch.org/whl/cu130@@ -99,9 +100,10 @@ function run_unit_test() {local test_basename=$(basename ${test_file} .py)local ut_log_name=${LOG_DIR}/unittest_cuda_${test_basename}.log- pytest -m "not skip_ci" -vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}+ pytest -m "not skip_ci" --cov=auto_round --cov-report= --cov-append -vs --disable-warnings --junitxml="${ut_log_name%.log}.xml" ${test_file} 2>&1 | tee ${ut_log_name}echo "##[endgroup]"done+ [ -f .coverage ] && cp .coverage "${LOG_DIR}/.coverage.part${test_part}"python ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/collect_result.py --test-type "CUDA Unit Tests" --log-pattern "unittest_cuda_test_*.log" --log-dir ${LOG_DIR} --summary-log ${SUMMARY_LOG}}@@ -129,9 +131,10 @@ function run_unit_test_llmc() {echo "##[group]Running ${test_file}..."local test_basename=$(basename ${test_file} .py)local ut_log_name=${LOG_DIR}/unittest_cuda_llmc_${test_basename}.log- pytest -m "not skip_ci" -vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}+ pytest -m "not skip_ci" --cov=auto_round --cov-report= --cov-append -vs --disable-warnings --junitxml="${ut_log_name%.log}.xml" ${test_file} 2>&1 | tee ${ut_log_name}echo "##[endgroup]"done+ [ -f .coverage ] && cp .coverage "${LOG_DIR}/.coverage.llmc"python ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/collect_result.py --test-type "CUDA LLMC Tests" --log-pattern "unittest_cuda_llmc_test_*.log" --log-dir ${LOG_DIR} --summary-log ${SUMMARY_LOG}}@@ -159,9 +162,10 @@ function run_unit_test_sglang() {echo "##[group]Running ${test_file}..."local test_basename=$(basename ${test_file} .py)local ut_log_name=${LOG_DIR}/unittest_cuda_sglang_${test_basename}.log- pytest -m "not skip_ci" -vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}+ pytest -m "not skip_ci" --cov=auto_round --cov-report= --cov-append -vs --disable-warnings --junitxml="${ut_log_name%.log}.xml" ${test_file} 2>&1 | tee ${ut_log_name}echo "##[endgroup]"done+ [ -f .coverage ] && cp .coverage "${LOG_DIR}/.coverage.sglang"python ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/collect_result.py --test-type "CUDA SGLang Tests" --log-pattern "unittest_cuda_sglang_test_*.log" --log-dir ${LOG_DIR} --summary-log ${SUMMARY_LOG}}@@ -189,9 +193,10 @@ function run_unit_test_vllm() {echo "##[group]Running ${test_file}..."local test_basename=$(basename ${test_file} .py)local ut_log_name=${LOG_DIR}/unittest_cuda_vllm_${test_basename}.log- pytest -m "not skip_ci" -vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}+ pytest -m "not skip_ci" --cov=auto_round --cov-report= --cov-append -vs --disable-warnings --junitxml="${ut_log_name%.log}.xml" ${test_file} 2>&1 | tee ${ut_log_name}echo "##[endgroup]"done+ [ -f .coverage ] && cp .coverage "${LOG_DIR}/.coverage.vllm"python ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/collect_result.py --test-type "CUDA VLLM Tests" --log-pattern "unittest_cuda_vllm_test_*.log" --log-dir ${LOG_DIR} --summary-log ${SUMMARY_LOG}}diff --git a/.azure-pipelines/scripts/ut/collect_result.py b/.azure-pipelines/scripts/ut/collect_result.pyindex 86813b1d2..31180b200 100644--- a/.azure-pipelines/scripts/ut/collect_result.py+++ b/.azure-pipelines/scripts/ut/collect_result.py@@ -1,8 +1,20 @@-"""Log analyzer for test results with summary generation."""+"""JUnit XML analyzer for test results with summary generation.++Each pytest invocation is expected to emit a JUnit XML report (via+``pytest --junitxml=<name>.xml``) alongside its ``<name>.log`` file. This+script enumerates the log files (which are always produced by ``tee``, even+when the process crashes), locates the sibling XML report and derives the+test status from the structured ``<testsuite>`` attributes instead of parsing+free-form log text.++If the XML report is missing or cannot be parsed (e.g. the process was killed+or segfaulted before pytest could write it), the corresponding test is marked+as FAILED.+"""import argparse-import reimport sys+import xml.etree.ElementTree as ETfrom dataclasses import dataclassfrom enum import Enum, autofrom pathlib import Path@@ -16,20 +28,29 @@ class TestStatus(Enum):NO_TESTS = auto()+@dataclass(frozen=True)+class TestCounts:+ """Aggregated test counts for a single test file."""++ passed: int = 0+ failed: int = 0+ skipped: int = 0+ total: int = 0++@dataclass(frozen=True)class TestResult:"""Represents a single test result."""name: strstatus: TestStatus- filename: strduration: str+ counts: TestCounts-class LogAnalyzer:- """Analyzes test log files and generates summary reports."""+class XmlAnalyzer:+ """Analyzes JUnit XML reports and generates summary results."""- TIME_PATTERN = re.compile(r"in\s+([\d.]+)s")PREFIX_PATTERNS = ("unittest_cuda_vlm_","unittest_cuda_vllm_",@@ -39,42 +60,12 @@ class LogAnalyzer:"unittest_",)- # pytest test logic failures: test ran but assertion/expectation failed- FAILURE_MARKERS = (- "FAILED",- "== FAILURES ==",- " failures",- "AssertionError",- )-- # runtime/system errors: process crashed, setup/teardown failed, or unhandled exception- ERROR_MARKERS = (- "Aborted",- "Killed",- "Segmentation fault",- "core dumped",- "Error:",- "ERROR:",- "== ERRORS ==",- " errors:",- "Exception:",- "Traceback ",- "Illegal instruction",- )-- PASS_MARKER = " passed"-- SKIP_MARKERS = (- " deselected",- " skipped",- )-def __init__(self, log_dir: Path, log_pattern: str = "*.log"):self.log_dir = Path(log_dir)self.log_pattern = log_patterndef analyze_all(self) -> list[TestResult]:- """Analyze all matching log files and return sorted results."""+ """Analyze all matching reports and return results sorted by log name."""search_path = self.log_dir / self.log_patternprint(f"Searching: {search_path}", file=sys.stderr)@@ -84,27 +75,20 @@ class LogAnalyzer:results = []for log_file in log_files:if log_file.is_file():- result = self._analyze_single(log_file)- results.append(result)+ results.append(self._analyze_single(log_file))return resultsdef _analyze_single(self, log_file: Path) -> TestResult:- content = self._read_log(log_file)+ xml_file = log_file.with_suffix(".xml")+ counts = self._parse_xml(xml_file)+ duration = self._parse_duration(xml_file)return TestResult(name=self._extract_test_name(log_file),- status=self._determine_status(content),- filename=log_file.name,- duration=self._extract_duration(content),+ status=self._determine_status(counts, xml_file),+ duration=duration,+ counts=counts if counts is not None else TestCounts(),)- def _read_log(self, path: Path) -> str:- try:- with open(path, "r", encoding="utf-8", errors="ignore") as f:- return f.read()- except OSError as e:- print(f"Warning: Cannot read {path} - {e}", file=sys.stderr)- return ""-def _extract_test_name(self, path: Path) -> str:name = path.stemfor prefix in self.PREFIX_PATTERNS:@@ -112,36 +96,63 @@ class LogAnalyzer:return name[len(prefix) :]return name- def _determine_status(self, content: str) -> TestStatus:- """Determine test status from log content."""- if any(marker in content for marker in self.FAILURE_MARKERS):- return TestStatus.FAILED- if any(marker in content for marker in self.ERROR_MARKERS):+ def _iter_testsuites(self, xml_file: Path):+ root = ET.parse(xml_file).getroot()+ if root.tag == "testsuite":+ yield root+ else: # <testsuites> wrapper+ yield from root.iter("testsuite")++ def _parse_xml(self, xml_file: Path) -> "TestCounts | None":+ """Return aggregated counts, or ``None`` if the report is unusable."""+ if not xml_file.is_file():+ return None+ try:+ total = failures = errors = skipped = 0+ for suite in self._iter_testsuites(xml_file):+ total += int(suite.get("tests", 0))+ failures += int(suite.get("failures", 0))+ errors += int(suite.get("errors", 0))+ skipped += int(suite.get("skipped", 0))+ except (ET.ParseError, ValueError, OSError) as e:+ print(f"Warning: cannot parse {xml_file} - {e}", file=sys.stderr)+ return None++ failed = failures + errors+ passed = max(total - failed - skipped, 0)+ return TestCounts(passed=passed, failed=failed, skipped=skipped, total=total)++ def _parse_duration(self, xml_file: Path) -> str:+ if not xml_file.is_file():+ return "00:00:00"+ try:+ seconds = 0.0+ for suite in self._iter_testsuites(xml_file):+ seconds += float(suite.get("time", 0.0))+ except (ET.ParseError, ValueError, OSError):+ return "00:00:00"+ total_seconds = int(seconds)+ hours, remainder = divmod(total_seconds, 3600)+ minutes, secs = divmod(remainder, 60)+ return f"{hours:02d}:{minutes:02d}:{secs:02d}"++ def _determine_status(self, counts: "TestCounts | None", xml_file: Path) -> TestStatus:+ # Missing or unparsable report => the run crashed before finishing.+ if counts is None:+ print(f"Warning: missing/invalid report {xml_file}", file=sys.stderr)return TestStatus.FAILED- if self.PASS_MARKER in content:- return TestStatus.PASSED- if any(marker in content for marker in self.SKIP_MARKERS):+ if counts.total == 0 or counts.total == counts.skipped:return TestStatus.NO_TESTS- return TestStatus.FAILED-- def _extract_duration(self, content: str) -> str:- last_match = None- for match in self.TIME_PATTERN.finditer(content):- last_match = match-- if last_match:- total_seconds = int(float(last_match.group(1)))- hours, remainder = divmod(total_seconds, 3600)- minutes, seconds = divmod(remainder, 60)- return f"{hours:02d}:{minutes:02d}:{seconds:02d}"- return "00:00:00"+ if counts.failed > 0:+ return TestStatus.FAILED+ return TestStatus.PASSEDclass ReportGenerator:"""Generates formatted test summary reports."""- WIDTHS = {"name": 35, "status": 10, "file": 50, "time": 10}- SEPARATOR = "=" * 120+ WIDTHS = {"name": 35, "status": 10, "passed": 9, "failed": 9, "skipped": 9, "time": 10}+ SEPARATOR = "=" * 100def __init__(self, output_path: Path):self.output_path = Path(output_path)@@ -189,7 +200,9 @@ class ReportGenerator:return (f"{'Test Case':<{self.WIDTHS['name']}} "f"{'Result':<{self.WIDTHS['status']}} "- f"{'Log File':<{self.WIDTHS['file']}} "+ f"{'Passed':<{self.WIDTHS['passed']}} "+ f"{'Failed':<{self.WIDTHS['failed']}} "+ f"{'Skipped':<{self.WIDTHS['skipped']}} "f"{'Time':<{self.WIDTHS['time']}}")@@ -197,7 +210,9 @@ class ReportGenerator:return (f"{'-' * 10:<{self.WIDTHS['name']}} "f"{'-' * 6:<{self.WIDTHS['status']}} "- f"{'-' * 8:<{self.WIDTHS['file']}} "+ f"{'-' * 6:<{self.WIDTHS['passed']}} "+ f"{'-' * 6:<{self.WIDTHS['failed']}} "+ f"{'-' * 6:<{self.WIDTHS['skipped']}} "f"{'-' * 4:<{self.WIDTHS['time']}}")@@ -205,17 +220,19 @@ class ReportGenerator:return (f"{result.name:<{self.WIDTHS['name']}} "f"{result.status.name:<{self.WIDTHS['status']}} "- f"{result.filename:<{self.WIDTHS['file']}} "+ f"{result.counts.passed:<{self.WIDTHS['passed']}} "+ f"{result.counts.failed:<{self.WIDTHS['failed']}} "+ f"{result.counts.skipped:<{self.WIDTHS['skipped']}} "f"{result.duration:<{self.WIDTHS['time']}}")def main():- parser = argparse.ArgumentParser(description="Analyze test logs and generate summary")+ parser = argparse.ArgumentParser(description="Analyze JUnit XML reports and generate summary")parser.add_argument("--test-type", required=True, help="Type of tests")- parser.add_argument("--log-dir", required=True, type=Path, help="Directory with logs")+ parser.add_argument("--log-dir", required=True, type=Path, help="Directory with logs/reports")parser.add_argument("--summary-log", required=True, type=Path, help="Output file")- parser.add_argument("--log-pattern", default="*.log", help="Glob pattern")+ parser.add_argument("--log-pattern", default="*.log", help="Glob pattern for log files")args = parser.parse_args()@@ -224,18 +241,24 @@ def main():sys.exit(1)try:- analyzer = LogAnalyzer(args.log_dir, args.log_pattern)+ analyzer = XmlAnalyzer(args.log_dir, args.log_pattern)results = analyzer.analyze_all()reporter = ReportGenerator(args.summary_log)reporter.generate(args.test_type, results)+ passed = sum(1 for r in results if r.status == TestStatus.PASSED)+ failed = sum(1 for r in results if r.status == TestStatus.FAILED)stats = {"total": len(results),- "passed": sum(1 for r in results if r.status == TestStatus.PASSED),- "failed": sum(1 for r in results if r.status == TestStatus.FAILED),+ "passed": passed,+ "failed": failed,+ "skipped": len(results) - passed - failed,}- print(f"Done: {stats['total']} tests, {stats['passed']} passed, {stats['failed']} failed")+ print(+ f"Done: {stats['total']} tests, {stats['passed']} passed, "+ f"{stats['failed']} failed, {stats['skipped']} skipped"+ )except Exception as e:print(f"Error: {e}", file=sys.stderr)diff --git a/.azure-pipelines/scripts/ut/run_ut.sh b/.azure-pipelines/scripts/ut/run_ut.shindex c1c3a2bbb..97f3c33c7 100644--- a/.azure-pipelines/scripts/ut/run_ut.sh+++ b/.azure-pipelines/scripts/ut/run_ut.sh@@ -24,7 +24,7 @@ function setup_environment() {# install latest gguf for ut testcd ~ || exit 1git clone -b master --quiet --single-branch https://github.com/ggml-org/llama.cpp.git && cd llama.cpp/gguf-py && uv pip install .-+cd /auto-round && uv pip install .export LD_LIBRARY_PATH=${HOME}/.venv/lib/:$LD_LIBRARY_PATH@@ -88,7 +88,7 @@ function run_unit_test() {numactl --physcpubind="${NUMA_CPUSET:-0-15}" --membind="${NUMA_NODE:-0}" \pytest --cov=auto_round --cov-report= --cov-append \- -vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}+ -vs --disable-warnings --junitxml="${ut_log_name%.log}.xml" ${test_file} 2>&1 | tee ${ut_log_name}echo "##[endgroup]"done}@@ -107,7 +107,7 @@ function run_inc_unit_test() {numactl --physcpubind="${NUMA_CPUSET:-0-15}" --membind="${NUMA_NODE:-0}" \pytest --cov=auto_round --cov-report= --cov-append \- -vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}+ -vs --disable-warnings --junitxml="${ut_log_name%.log}.xml" ${test_file} 2>&1 | tee ${ut_log_name}echo "##[endgroup]"done}@@ -128,7 +128,7 @@ function run_llmc_unit_test() {numactl --physcpubind="${NUMA_CPUSET:-0-15}" --membind="${NUMA_NODE:-0}" \pytest --cov=auto_round --cov-report= --cov-append \- -vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}+ -vs --disable-warnings --junitxml="${ut_log_name%.log}.xml" ${test_file} 2>&1 | tee ${ut_log_name}echo "##[endgroup]"done}diff --git a/.azure-pipelines/scripts/ut/run_ut_hpu.sh b/.azure-pipelines/scripts/ut/run_ut_hpu.shindex 46c731845..9599946c7 100644--- a/.azure-pipelines/scripts/ut/run_ut_hpu.sh+++ b/.azure-pipelines/scripts/ut/run_ut_hpu.sh@@ -35,14 +35,14 @@ function run_unit_test() {local ut_log_name="${LOG_DIR}/unittest_lazy_${test_basename}.log"PT_HPU_LAZY_MODE=1 pytest --cov="${auto_round_path}" \--cov-report= --cov-append -vs --disable-warnings \- ${test_file} 2>&1 | tee ${ut_log_name}+ --junitxml="${ut_log_name%.log}.xml" ${test_file} 2>&1 | tee ${ut_log_name}echo "##[endgroup]"echo "##[group]Running ${test_file} in HPU compile mode..."local ut_log_name="${LOG_DIR}/unittest_compile_${test_basename}.log"PT_HPU_LAZY_MODE=0 pytest --mode compile --cov="${auto_round_path}" \--cov-report= --cov-append -vs --disable-warnings \- ${test_file} 2>&1 | tee ${ut_log_name}+ --junitxml="${ut_log_name%.log}.xml" ${test_file} 2>&1 | tee ${ut_log_name}echo "##[endgroup]"done}diff --git a/.azure-pipelines/scripts/ut/run_ut_xpu.sh b/.azure-pipelines/scripts/ut/run_ut_xpu.shindex 30b06bdff..5705cca29 100644--- a/.azure-pipelines/scripts/ut/run_ut_xpu.sh+++ b/.azure-pipelines/scripts/ut/run_ut_xpu.sh@@ -40,7 +40,7 @@ function run_unit_test() {local ut_log_name="${LOG_DIR}/unittest_ark_${test_basename}.log"numactl --physcpubind="${NUMA_CPUSET:-0-27}" --membind="${NUMA_NODE:-0}" \pytest --cov="${auto_round_path}" --cov-report= --cov-append -vs --disable-warnings \- ${test_file} 2>&1 | tee ${ut_log_name}+ --junitxml="${ut_log_name%.log}.xml" ${test_file} 2>&1 | tee ${ut_log_name}echo "##[endgroup]"done@@ -51,13 +51,13 @@ function run_unit_test() {local ut_log_name="${LOG_DIR}/unittest_xpu_${test_basename}.log"numactl --physcpubind="${NUMA_CPUSET:-0-27}" --membind="${NUMA_NODE:-0}" \pytest --cov="${auto_round_path}" --cov-report= --cov-append -vs --disable-warnings \- ${test_file} 2>&1 | tee ${ut_log_name}+ --junitxml="${ut_log_name%.log}.xml" ${test_file} 2>&1 | tee ${ut_log_name}echo "##[endgroup]"done}-function run_unit_test_vllm() {- echo "##[group]set up vllm UT env..."+function run_unit_test_llmc() {+ echo "##[group]set up llmc UT env..."BUILD_TYPE="nightly" uv pip install -r ./test_xpu/requirements_llmc.txtuv pip listecho "##[endgroup]"@@ -67,11 +67,11 @@ function run_unit_test_vllm() {for test_file in $(find ./test_xpu -name "test_llmc_integration.py" | sort); dolocal test_basename=$(basename ${test_file} .py)- echo "##[group]Running xpu vLLM ${test_file}..."+ echo "##[group]Running xpu llmc ${test_file}..."local ut_log_name="${LOG_DIR}/unittest_xpu_${test_basename}.log"numactl --physcpubind="${NUMA_CPUSET:-0-27}" --membind="${NUMA_NODE:-0}" \pytest --cov="${auto_round_path}" --cov-report= --cov-append -vs --disable-warnings \- ${test_file} 2>&1 | tee ${ut_log_name}+ --junitxml="${ut_log_name%.log}.xml" ${test_file} 2>&1 | tee ${ut_log_name}echo "##[endgroup]"done}@@ -110,7 +110,7 @@ function print_coverage() {function main() {setup_environmentrun_unit_test- run_unit_test_vllm+ run_unit_test_llmccollect_logprint_coverageprint_summarydiff --git a/.azure-pipelines/unit-test-cuda.yml b/.azure-pipelines/unit-test-cuda.ymlindex bf66fdac9..cb2253d37 100644--- a/.azure-pipelines/unit-test-cuda.yml+++ b/.azure-pipelines/unit-test-cuda.yml@@ -70,8 +70,6 @@ stages:inputs:targetType: "inline"script: |- export PYTHONUNBUFFERED=1- cd ${BUILD_SOURCESDIRECTORY}CUDA_VERSION="13.0"python .azure-pipelines/scripts/cuda_unit_test/runpod_manager.py \@@ -82,6 +80,8 @@ stages:--part ${{ pair.value.PART }} \--cuda_version "${CUDA_VERSION}" \--env AZP_URL=$(System.CollectionUri) AZP_TOKEN=$(AZP_DEVOPS_PAT) AZP_POOL=$(POOL_NAME) AZP_AGENT_NAME="$(AGENT_NAME)-part${{ pair.value.PART }}"+ env:+ PYTHONUNBUFFERED: '1'- task: Bash@3displayName: "Wait for Pod to be Online"@@ -89,12 +89,12 @@ stages:inputs:targetType: "inline"script: |- cd ${BUILD_SOURCESDIRECTORY}- export PYTHONUNBUFFERED=1python .azure-pipelines/scripts/cuda_unit_test/runpod_manager.py \--action wait \--api_key $(RUNPOD_API_KEY) \--name "ci-$(Build.BuildId)-${{ pair.value.PART }}"+ env:+ PYTHONUNBUFFERED: '1'- task: Bash@3displayName: "Wait for Agent to be Online"@@ -102,14 +102,14 @@ stages:inputs:targetType: "inline"script: |- cd ${BUILD_SOURCESDIRECTORY}- export PYTHONUNBUFFERED=1python .azure-pipelines/scripts/cuda_unit_test/azure_agent.py \--action wait \--url "$(System.CollectionUri)" \--pat "$(AZP_DEVOPS_PAT)" \--pool "$(POOL_NAME)" \--agent "$(AGENT_NAME)-part${{ pair.value.PART }}"+ env:+ PYTHONUNBUFFERED: '1'- stage: GPU_Test_${{ pair.key }}displayName: "GPU Test ${{ pair.value.PART }}"@@ -152,6 +152,14 @@ stages:fidisplayName: "Run GPU Tests"+ - task: PublishPipelineArtifact@1+ condition: succeeded()+ inputs:+ targetPath: $(Build.SourcesDirectory)/log_dir+ artifact: "ut-${{ pair.value.PART }}_coverage"+ publishLocation: "pipeline"+ displayName: "Publish UT coverage artifact"+- script: |cd .azure-pipelines/scripts/cuda_unit_testuv run monitor_gpu.py stop@@ -191,11 +199,12 @@ stages:targetType: "inline"script: |echo "Terminating Pod: ci-$(Build.BuildId)-${{ pair.value.PART }}"- export PYTHONUNBUFFERED=1python .azure-pipelines/scripts/cuda_unit_test/runpod_manager.py \--action terminate \--api_key $(RUNPOD_API_KEY) \--name "ci-$(Build.BuildId)-${{ pair.value.PART }}"+ env:+ PYTHONUNBUFFERED: '1'- task: Bash@3displayName: "Deregister Agent from Pool"@@ -204,10 +213,62 @@ stages:targetType: "inline"script: |cd ${BUILD_SOURCESDIRECTORY}- export PYTHONUNBUFFERED=1python .azure-pipelines/scripts/cuda_unit_test/azure_agent.py \--action deregister \--url "$(System.CollectionUri)" \--pat "$(AZP_DEVOPS_PAT)" \--pool "$(POOL_NAME)" \--agent "$(AGENT_NAME)-part${{ pair.value.PART }}"+ env:+ PYTHONUNBUFFERED: '1'++ - stage: Coverage+ displayName: "Collect Coverage"+ pool:+ vmImage: "ubuntu-latest"+ dependsOn:+ - ${{ each pair in parameters.matrix }}:+ - GPU_Test_${{ pair.key }}+ - Cleanup_Verification_${{ pair.key }}+ variables:+ UPLOAD_PATH: $(Build.SourcesDirectory)/log_dir+ DOWNLOAD_PATH: $(Build.SourcesDirectory)/log_dir+ ARTIFACT_NAME: "UT_coverage_report"+ jobs:+ - job: CollectDatafiles+ steps:+ - checkout: self++ - task: DownloadPipelineArtifact@2+ inputs:+ patterns: 'ut-*_coverage/.coverage.*'+ path: $(DOWNLOAD_PATH)++ - task: UsePythonVersion@0+ inputs:+ versionSpec: '3.12'+ displayName: 'Use Python 3.12'++ - script: |+ cd ${BUILD_SOURCESDIRECTORY}+ pip install -U pip setuptools uv+ uv pip install -r requirements.txt --extra-index-url https://download.pytorch.org/whl/cpu+ uv pip install .+ pip list+ bash .azure-pipelines/scripts/ut/collect_log.sh+ env:+ PYTHONUNBUFFERED: '1'+ UV_NO_PROGRESS: '1'+ UV_SYSTEM_PYTHON: '1'+ displayName: "Collect UT Coverage"++ - task: PublishPipelineArtifact@1+ condition: succeededOrFailed()+ inputs:+ targetPath: $(UPLOAD_PATH)/coverage_PR+ artifact: $(ARTIFACT_NAME)+ publishLocation: "pipeline"++ - task: PublishCodeCoverageResults@2+ inputs:+ summaryFileLocation: $(Build.SourcesDirectory)/log_dir/coverage_PR/coverage.xml