Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Add CUDA CI coverage (#2074)

Sun, Xuehao6ff414b15728
diff --git a/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh b/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh
index 27ae8e2bd..68b85dffa 100644
--- a/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh
+++ b/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh
@@ -20,7 +20,7 @@ done
source ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/change_color.sh
-LOG_DIR="${BUILD_SOURCESDIRECTORY}/ut_log_dir"
+LOG_DIR="${BUILD_SOURCESDIRECTORY}/log_dir"
mkdir -p "${LOG_DIR}"
SUMMARY_LOG="${LOG_DIR}/results_summary.log"
@@ -51,10 +51,10 @@ function print_summary() {
function check_storage_usage() {
echo "##[group]check storage usage..."
df -h
- du -sh "${BUILD_SOURCESDIRECTORY}"
- du -sh /root/.cache/huggingface
- du -sh /root/.cache/huggingface/hub/*
- du -sh /root/.venv
+ du -sh "${BUILD_SOURCESDIRECTORY}" || true
+ du -sh /root/.cache/huggingface || true
+ du -sh /root/.cache/huggingface/hub/* || true
+ du -sh /root/.venv || true
echo "##[endgroup]"
}
@@ -68,6 +68,7 @@ function run_unit_test() {
uv pip install -r test/test_cuda/requirements.txt
uv pip install -r test/test_cuda/requirements_diffusion.txt
uv pip install -U transformers chardet
+ uv pip install -U pytest-cov
uv pip install kernels==0.15.2 # For sm120: https://github.com/huggingface/transformers/blob/v5.13.1/setup.py#L93
uv pip uninstall torch torchvision
uv pip install torch==2.13.0 torchvision torchao --index-url https://download.pytorch.org/whl/cu130
@@ -99,9 +100,10 @@ function run_unit_test() {
local test_basename=$(basename ${test_file} .py)
local ut_log_name=${LOG_DIR}/unittest_cuda_${test_basename}.log
- pytest -m "not skip_ci" -vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}
+ pytest -m "not skip_ci" --cov=auto_round --cov-report= --cov-append -vs --disable-warnings --junitxml="${ut_log_name%.log}.xml" ${test_file} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
done
+ [ -f .coverage ] && cp .coverage "${LOG_DIR}/.coverage.part${test_part}"
python ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/collect_result.py --test-type "CUDA Unit Tests" --log-pattern "unittest_cuda_test_*.log" --log-dir ${LOG_DIR} --summary-log ${SUMMARY_LOG}
}
@@ -129,9 +131,10 @@ function run_unit_test_llmc() {
echo "##[group]Running ${test_file}..."
local test_basename=$(basename ${test_file} .py)
local ut_log_name=${LOG_DIR}/unittest_cuda_llmc_${test_basename}.log
- pytest -m "not skip_ci" -vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}
+ pytest -m "not skip_ci" --cov=auto_round --cov-report= --cov-append -vs --disable-warnings --junitxml="${ut_log_name%.log}.xml" ${test_file} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
done
+ [ -f .coverage ] && cp .coverage "${LOG_DIR}/.coverage.llmc"
python ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/collect_result.py --test-type "CUDA LLMC Tests" --log-pattern "unittest_cuda_llmc_test_*.log" --log-dir ${LOG_DIR} --summary-log ${SUMMARY_LOG}
}
@@ -159,9 +162,10 @@ function run_unit_test_sglang() {
echo "##[group]Running ${test_file}..."
local test_basename=$(basename ${test_file} .py)
local ut_log_name=${LOG_DIR}/unittest_cuda_sglang_${test_basename}.log
- pytest -m "not skip_ci" -vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}
+ pytest -m "not skip_ci" --cov=auto_round --cov-report= --cov-append -vs --disable-warnings --junitxml="${ut_log_name%.log}.xml" ${test_file} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
done
+ [ -f .coverage ] && cp .coverage "${LOG_DIR}/.coverage.sglang"
python ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/collect_result.py --test-type "CUDA SGLang Tests" --log-pattern "unittest_cuda_sglang_test_*.log" --log-dir ${LOG_DIR} --summary-log ${SUMMARY_LOG}
}
@@ -189,9 +193,10 @@ function run_unit_test_vllm() {
echo "##[group]Running ${test_file}..."
local test_basename=$(basename ${test_file} .py)
local ut_log_name=${LOG_DIR}/unittest_cuda_vllm_${test_basename}.log
- pytest -m "not skip_ci" -vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}
+ pytest -m "not skip_ci" --cov=auto_round --cov-report= --cov-append -vs --disable-warnings --junitxml="${ut_log_name%.log}.xml" ${test_file} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
done
+ [ -f .coverage ] && cp .coverage "${LOG_DIR}/.coverage.vllm"
python ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/collect_result.py --test-type "CUDA VLLM Tests" --log-pattern "unittest_cuda_vllm_test_*.log" --log-dir ${LOG_DIR} --summary-log ${SUMMARY_LOG}
}
diff --git a/.azure-pipelines/scripts/ut/collect_result.py b/.azure-pipelines/scripts/ut/collect_result.py
index 86813b1d2..31180b200 100644
--- a/.azure-pipelines/scripts/ut/collect_result.py
+++ b/.azure-pipelines/scripts/ut/collect_result.py
@@ -1,8 +1,20 @@
-"""Log analyzer for test results with summary generation."""
+"""JUnit XML analyzer for test results with summary generation.
+
+Each pytest invocation is expected to emit a JUnit XML report (via
+``pytest --junitxml=<name>.xml``) alongside its ``<name>.log`` file. This
+script enumerates the log files (which are always produced by ``tee``, even
+when the process crashes), locates the sibling XML report and derives the
+test status from the structured ``<testsuite>`` attributes instead of parsing
+free-form log text.
+
+If the XML report is missing or cannot be parsed (e.g. the process was killed
+or segfaulted before pytest could write it), the corresponding test is marked
+as FAILED.
+"""
import argparse
-import re
import sys
+import xml.etree.ElementTree as ET
from dataclasses import dataclass
from enum import Enum, auto
from pathlib import Path
@@ -16,20 +28,29 @@ class TestStatus(Enum):
NO_TESTS = auto()
+@dataclass(frozen=True)
+class TestCounts:
+ """Aggregated test counts for a single test file."""
+
+ passed: int = 0
+ failed: int = 0
+ skipped: int = 0
+ total: int = 0
+
+
@dataclass(frozen=True)
class TestResult:
"""Represents a single test result."""
name: str
status: TestStatus
- filename: str
duration: str
+ counts: TestCounts
-class LogAnalyzer:
- """Analyzes test log files and generates summary reports."""
+class XmlAnalyzer:
+ """Analyzes JUnit XML reports and generates summary results."""
- TIME_PATTERN = re.compile(r"in\s+([\d.]+)s")
PREFIX_PATTERNS = (
"unittest_cuda_vlm_",
"unittest_cuda_vllm_",
@@ -39,42 +60,12 @@ class LogAnalyzer:
"unittest_",
)
- # pytest test logic failures: test ran but assertion/expectation failed
- FAILURE_MARKERS = (
- "FAILED",
- "== FAILURES ==",
- " failures",
- "AssertionError",
- )
-
- # runtime/system errors: process crashed, setup/teardown failed, or unhandled exception
- ERROR_MARKERS = (
- "Aborted",
- "Killed",
- "Segmentation fault",
- "core dumped",
- "Error:",
- "ERROR:",
- "== ERRORS ==",
- " errors:",
- "Exception:",
- "Traceback ",
- "Illegal instruction",
- )
-
- PASS_MARKER = " passed"
-
- SKIP_MARKERS = (
- " deselected",
- " skipped",
- )
-
def __init__(self, log_dir: Path, log_pattern: str = "*.log"):
self.log_dir = Path(log_dir)
self.log_pattern = log_pattern
def analyze_all(self) -> list[TestResult]:
- """Analyze all matching log files and return sorted results."""
+ """Analyze all matching reports and return results sorted by log name."""
search_path = self.log_dir / self.log_pattern
print(f"Searching: {search_path}", file=sys.stderr)
@@ -84,27 +75,20 @@ class LogAnalyzer:
results = []
for log_file in log_files:
if log_file.is_file():
- result = self._analyze_single(log_file)
- results.append(result)
+ results.append(self._analyze_single(log_file))
return results
def _analyze_single(self, log_file: Path) -> TestResult:
- content = self._read_log(log_file)
+ xml_file = log_file.with_suffix(".xml")
+ counts = self._parse_xml(xml_file)
+ duration = self._parse_duration(xml_file)
return TestResult(
name=self._extract_test_name(log_file),
- status=self._determine_status(content),
- filename=log_file.name,
- duration=self._extract_duration(content),
+ status=self._determine_status(counts, xml_file),
+ duration=duration,
+ counts=counts if counts is not None else TestCounts(),
)
- def _read_log(self, path: Path) -> str:
- try:
- with open(path, "r", encoding="utf-8", errors="ignore") as f:
- return f.read()
- except OSError as e:
- print(f"Warning: Cannot read {path} - {e}", file=sys.stderr)
- return ""
-
def _extract_test_name(self, path: Path) -> str:
name = path.stem
for prefix in self.PREFIX_PATTERNS:
@@ -112,36 +96,63 @@ class LogAnalyzer:
return name[len(prefix) :]
return name
- def _determine_status(self, content: str) -> TestStatus:
- """Determine test status from log content."""
- if any(marker in content for marker in self.FAILURE_MARKERS):
- return TestStatus.FAILED
- if any(marker in content for marker in self.ERROR_MARKERS):
+ def _iter_testsuites(self, xml_file: Path):
+ root = ET.parse(xml_file).getroot()
+ if root.tag == "testsuite":
+ yield root
+ else: # <testsuites> wrapper
+ yield from root.iter("testsuite")
+
+ def _parse_xml(self, xml_file: Path) -> "TestCounts | None":
+ """Return aggregated counts, or ``None`` if the report is unusable."""
+ if not xml_file.is_file():
+ return None
+ try:
+ total = failures = errors = skipped = 0
+ for suite in self._iter_testsuites(xml_file):
+ total += int(suite.get("tests", 0))
+ failures += int(suite.get("failures", 0))
+ errors += int(suite.get("errors", 0))
+ skipped += int(suite.get("skipped", 0))
+ except (ET.ParseError, ValueError, OSError) as e:
+ print(f"Warning: cannot parse {xml_file} - {e}", file=sys.stderr)
+ return None
+
+ failed = failures + errors
+ passed = max(total - failed - skipped, 0)
+ return TestCounts(passed=passed, failed=failed, skipped=skipped, total=total)
+
+ def _parse_duration(self, xml_file: Path) -> str:
+ if not xml_file.is_file():
+ return "00:00:00"
+ try:
+ seconds = 0.0
+ for suite in self._iter_testsuites(xml_file):
+ seconds += float(suite.get("time", 0.0))
+ except (ET.ParseError, ValueError, OSError):
+ return "00:00:00"
+ total_seconds = int(seconds)
+ hours, remainder = divmod(total_seconds, 3600)
+ minutes, secs = divmod(remainder, 60)
+ return f"{hours:02d}:{minutes:02d}:{secs:02d}"
+
+ def _determine_status(self, counts: "TestCounts | None", xml_file: Path) -> TestStatus:
+ # Missing or unparsable report => the run crashed before finishing.
+ if counts is None:
+ print(f"Warning: missing/invalid report {xml_file}", file=sys.stderr)
return TestStatus.FAILED
- if self.PASS_MARKER in content:
- return TestStatus.PASSED
- if any(marker in content for marker in self.SKIP_MARKERS):
+ if counts.total == 0 or counts.total == counts.skipped:
return TestStatus.NO_TESTS
- return TestStatus.FAILED
-
- def _extract_duration(self, content: str) -> str:
- last_match = None
- for match in self.TIME_PATTERN.finditer(content):
- last_match = match
-
- if last_match:
- total_seconds = int(float(last_match.group(1)))
- hours, remainder = divmod(total_seconds, 3600)
- minutes, seconds = divmod(remainder, 60)
- return f"{hours:02d}:{minutes:02d}:{seconds:02d}"
- return "00:00:00"
+ if counts.failed > 0:
+ return TestStatus.FAILED
+ return TestStatus.PASSED
class ReportGenerator:
"""Generates formatted test summary reports."""
- WIDTHS = {"name": 35, "status": 10, "file": 50, "time": 10}
- SEPARATOR = "=" * 120
+ WIDTHS = {"name": 35, "status": 10, "passed": 9, "failed": 9, "skipped": 9, "time": 10}
+ SEPARATOR = "=" * 100
def __init__(self, output_path: Path):
self.output_path = Path(output_path)
@@ -189,7 +200,9 @@ class ReportGenerator:
return (
f"{'Test Case':<{self.WIDTHS['name']}} "
f"{'Result':<{self.WIDTHS['status']}} "
- f"{'Log File':<{self.WIDTHS['file']}} "
+ f"{'Passed':<{self.WIDTHS['passed']}} "
+ f"{'Failed':<{self.WIDTHS['failed']}} "
+ f"{'Skipped':<{self.WIDTHS['skipped']}} "
f"{'Time':<{self.WIDTHS['time']}}"
)
@@ -197,7 +210,9 @@ class ReportGenerator:
return (
f"{'-' * 10:<{self.WIDTHS['name']}} "
f"{'-' * 6:<{self.WIDTHS['status']}} "
- f"{'-' * 8:<{self.WIDTHS['file']}} "
+ f"{'-' * 6:<{self.WIDTHS['passed']}} "
+ f"{'-' * 6:<{self.WIDTHS['failed']}} "
+ f"{'-' * 6:<{self.WIDTHS['skipped']}} "
f"{'-' * 4:<{self.WIDTHS['time']}}"
)
@@ -205,17 +220,19 @@ class ReportGenerator:
return (
f"{result.name:<{self.WIDTHS['name']}} "
f"{result.status.name:<{self.WIDTHS['status']}} "
- f"{result.filename:<{self.WIDTHS['file']}} "
+ f"{result.counts.passed:<{self.WIDTHS['passed']}} "
+ f"{result.counts.failed:<{self.WIDTHS['failed']}} "
+ f"{result.counts.skipped:<{self.WIDTHS['skipped']}} "
f"{result.duration:<{self.WIDTHS['time']}}"
)
def main():
- parser = argparse.ArgumentParser(description="Analyze test logs and generate summary")
+ parser = argparse.ArgumentParser(description="Analyze JUnit XML reports and generate summary")
parser.add_argument("--test-type", required=True, help="Type of tests")
- parser.add_argument("--log-dir", required=True, type=Path, help="Directory with logs")
+ parser.add_argument("--log-dir", required=True, type=Path, help="Directory with logs/reports")
parser.add_argument("--summary-log", required=True, type=Path, help="Output file")
- parser.add_argument("--log-pattern", default="*.log", help="Glob pattern")
+ parser.add_argument("--log-pattern", default="*.log", help="Glob pattern for log files")
args = parser.parse_args()
@@ -224,18 +241,24 @@ def main():
sys.exit(1)
try:
- analyzer = LogAnalyzer(args.log_dir, args.log_pattern)
+ analyzer = XmlAnalyzer(args.log_dir, args.log_pattern)
results = analyzer.analyze_all()
reporter = ReportGenerator(args.summary_log)
reporter.generate(args.test_type, results)
+ passed = sum(1 for r in results if r.status == TestStatus.PASSED)
+ failed = sum(1 for r in results if r.status == TestStatus.FAILED)
stats = {
"total": len(results),
- "passed": sum(1 for r in results if r.status == TestStatus.PASSED),
- "failed": sum(1 for r in results if r.status == TestStatus.FAILED),
+ "passed": passed,
+ "failed": failed,
+ "skipped": len(results) - passed - failed,
}
- print(f"Done: {stats['total']} tests, {stats['passed']} passed, {stats['failed']} failed")
+ print(
+ f"Done: {stats['total']} tests, {stats['passed']} passed, "
+ f"{stats['failed']} failed, {stats['skipped']} skipped"
+ )
except Exception as e:
print(f"Error: {e}", file=sys.stderr)
diff --git a/.azure-pipelines/scripts/ut/run_ut.sh b/.azure-pipelines/scripts/ut/run_ut.sh
index c1c3a2bbb..97f3c33c7 100644
--- a/.azure-pipelines/scripts/ut/run_ut.sh
+++ b/.azure-pipelines/scripts/ut/run_ut.sh
@@ -24,7 +24,7 @@ function setup_environment() {
# install latest gguf for ut test
cd ~ || exit 1
git clone -b master --quiet --single-branch https://github.com/ggml-org/llama.cpp.git && cd llama.cpp/gguf-py && uv pip install .
-
+
cd /auto-round && uv pip install .
export LD_LIBRARY_PATH=${HOME}/.venv/lib/:$LD_LIBRARY_PATH
@@ -88,7 +88,7 @@ function run_unit_test() {
numactl --physcpubind="${NUMA_CPUSET:-0-15}" --membind="${NUMA_NODE:-0}" \
pytest --cov=auto_round --cov-report= --cov-append \
- -vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}
+ -vs --disable-warnings --junitxml="${ut_log_name%.log}.xml" ${test_file} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
done
}
@@ -107,7 +107,7 @@ function run_inc_unit_test() {
numactl --physcpubind="${NUMA_CPUSET:-0-15}" --membind="${NUMA_NODE:-0}" \
pytest --cov=auto_round --cov-report= --cov-append \
- -vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}
+ -vs --disable-warnings --junitxml="${ut_log_name%.log}.xml" ${test_file} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
done
}
@@ -128,7 +128,7 @@ function run_llmc_unit_test() {
numactl --physcpubind="${NUMA_CPUSET:-0-15}" --membind="${NUMA_NODE:-0}" \
pytest --cov=auto_round --cov-report= --cov-append \
- -vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}
+ -vs --disable-warnings --junitxml="${ut_log_name%.log}.xml" ${test_file} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
done
}
diff --git a/.azure-pipelines/scripts/ut/run_ut_hpu.sh b/.azure-pipelines/scripts/ut/run_ut_hpu.sh
index 46c731845..9599946c7 100644
--- a/.azure-pipelines/scripts/ut/run_ut_hpu.sh
+++ b/.azure-pipelines/scripts/ut/run_ut_hpu.sh
@@ -35,14 +35,14 @@ function run_unit_test() {
local ut_log_name="${LOG_DIR}/unittest_lazy_${test_basename}.log"
PT_HPU_LAZY_MODE=1 pytest --cov="${auto_round_path}" \
--cov-report= --cov-append -vs --disable-warnings \
- ${test_file} 2>&1 | tee ${ut_log_name}
+ --junitxml="${ut_log_name%.log}.xml" ${test_file} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
echo "##[group]Running ${test_file} in HPU compile mode..."
local ut_log_name="${LOG_DIR}/unittest_compile_${test_basename}.log"
PT_HPU_LAZY_MODE=0 pytest --mode compile --cov="${auto_round_path}" \
--cov-report= --cov-append -vs --disable-warnings \
- ${test_file} 2>&1 | tee ${ut_log_name}
+ --junitxml="${ut_log_name%.log}.xml" ${test_file} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
done
}
diff --git a/.azure-pipelines/scripts/ut/run_ut_xpu.sh b/.azure-pipelines/scripts/ut/run_ut_xpu.sh
index 30b06bdff..5705cca29 100644
--- a/.azure-pipelines/scripts/ut/run_ut_xpu.sh
+++ b/.azure-pipelines/scripts/ut/run_ut_xpu.sh
@@ -40,7 +40,7 @@ function run_unit_test() {
local ut_log_name="${LOG_DIR}/unittest_ark_${test_basename}.log"
numactl --physcpubind="${NUMA_CPUSET:-0-27}" --membind="${NUMA_NODE:-0}" \
pytest --cov="${auto_round_path}" --cov-report= --cov-append -vs --disable-warnings \
- ${test_file} 2>&1 | tee ${ut_log_name}
+ --junitxml="${ut_log_name%.log}.xml" ${test_file} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
done
@@ -51,13 +51,13 @@ function run_unit_test() {
local ut_log_name="${LOG_DIR}/unittest_xpu_${test_basename}.log"
numactl --physcpubind="${NUMA_CPUSET:-0-27}" --membind="${NUMA_NODE:-0}" \
pytest --cov="${auto_round_path}" --cov-report= --cov-append -vs --disable-warnings \
- ${test_file} 2>&1 | tee ${ut_log_name}
+ --junitxml="${ut_log_name%.log}.xml" ${test_file} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
done
}
-function run_unit_test_vllm() {
- echo "##[group]set up vllm UT env..."
+function run_unit_test_llmc() {
+ echo "##[group]set up llmc UT env..."
BUILD_TYPE="nightly" uv pip install -r ./test_xpu/requirements_llmc.txt
uv pip list
echo "##[endgroup]"
@@ -67,11 +67,11 @@ function run_unit_test_vllm() {
for test_file in $(find ./test_xpu -name "test_llmc_integration.py" | sort); do
local test_basename=$(basename ${test_file} .py)
- echo "##[group]Running xpu vLLM ${test_file}..."
+ echo "##[group]Running xpu llmc ${test_file}..."
local ut_log_name="${LOG_DIR}/unittest_xpu_${test_basename}.log"
numactl --physcpubind="${NUMA_CPUSET:-0-27}" --membind="${NUMA_NODE:-0}" \
pytest --cov="${auto_round_path}" --cov-report= --cov-append -vs --disable-warnings \
- ${test_file} 2>&1 | tee ${ut_log_name}
+ --junitxml="${ut_log_name%.log}.xml" ${test_file} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
done
}
@@ -110,7 +110,7 @@ function print_coverage() {
function main() {
setup_environment
run_unit_test
- run_unit_test_vllm
+ run_unit_test_llmc
collect_log
print_coverage
print_summary
diff --git a/.azure-pipelines/unit-test-cuda.yml b/.azure-pipelines/unit-test-cuda.yml
index bf66fdac9..cb2253d37 100644
--- a/.azure-pipelines/unit-test-cuda.yml
+++ b/.azure-pipelines/unit-test-cuda.yml
@@ -70,8 +70,6 @@ stages:
inputs:
targetType: "inline"
script: |
- export PYTHONUNBUFFERED=1
- cd ${BUILD_SOURCESDIRECTORY}
CUDA_VERSION="13.0"
python .azure-pipelines/scripts/cuda_unit_test/runpod_manager.py \
@@ -82,6 +80,8 @@ stages:
--part ${{ pair.value.PART }} \
--cuda_version "${CUDA_VERSION}" \
--env AZP_URL=$(System.CollectionUri) AZP_TOKEN=$(AZP_DEVOPS_PAT) AZP_POOL=$(POOL_NAME) AZP_AGENT_NAME="$(AGENT_NAME)-part${{ pair.value.PART }}"
+ env:
+ PYTHONUNBUFFERED: '1'
- task: Bash@3
displayName: "Wait for Pod to be Online"
@@ -89,12 +89,12 @@ stages:
inputs:
targetType: "inline"
script: |
- cd ${BUILD_SOURCESDIRECTORY}
- export PYTHONUNBUFFERED=1
python .azure-pipelines/scripts/cuda_unit_test/runpod_manager.py \
--action wait \
--api_key $(RUNPOD_API_KEY) \
--name "ci-$(Build.BuildId)-${{ pair.value.PART }}"
+ env:
+ PYTHONUNBUFFERED: '1'
- task: Bash@3
displayName: "Wait for Agent to be Online"
@@ -102,14 +102,14 @@ stages:
inputs:
targetType: "inline"
script: |
- cd ${BUILD_SOURCESDIRECTORY}
- export PYTHONUNBUFFERED=1
python .azure-pipelines/scripts/cuda_unit_test/azure_agent.py \
--action wait \
--url "$(System.CollectionUri)" \
--pat "$(AZP_DEVOPS_PAT)" \
--pool "$(POOL_NAME)" \
--agent "$(AGENT_NAME)-part${{ pair.value.PART }}"
+ env:
+ PYTHONUNBUFFERED: '1'
- stage: GPU_Test_${{ pair.key }}
displayName: "GPU Test ${{ pair.value.PART }}"
@@ -152,6 +152,14 @@ stages:
fi
displayName: "Run GPU Tests"
+ - task: PublishPipelineArtifact@1
+ condition: succeeded()
+ inputs:
+ targetPath: $(Build.SourcesDirectory)/log_dir
+ artifact: "ut-${{ pair.value.PART }}_coverage"
+ publishLocation: "pipeline"
+ displayName: "Publish UT coverage artifact"
+
- script: |
cd .azure-pipelines/scripts/cuda_unit_test
uv run monitor_gpu.py stop
@@ -191,11 +199,12 @@ stages:
targetType: "inline"
script: |
echo "Terminating Pod: ci-$(Build.BuildId)-${{ pair.value.PART }}"
- export PYTHONUNBUFFERED=1
python .azure-pipelines/scripts/cuda_unit_test/runpod_manager.py \
--action terminate \
--api_key $(RUNPOD_API_KEY) \
--name "ci-$(Build.BuildId)-${{ pair.value.PART }}"
+ env:
+ PYTHONUNBUFFERED: '1'
- task: Bash@3
displayName: "Deregister Agent from Pool"
@@ -204,10 +213,62 @@ stages:
targetType: "inline"
script: |
cd ${BUILD_SOURCESDIRECTORY}
- export PYTHONUNBUFFERED=1
python .azure-pipelines/scripts/cuda_unit_test/azure_agent.py \
--action deregister \
--url "$(System.CollectionUri)" \
--pat "$(AZP_DEVOPS_PAT)" \
--pool "$(POOL_NAME)" \
--agent "$(AGENT_NAME)-part${{ pair.value.PART }}"
+ env:
+ PYTHONUNBUFFERED: '1'
+
+ - stage: Coverage
+ displayName: "Collect Coverage"
+ pool:
+ vmImage: "ubuntu-latest"
+ dependsOn:
+ - ${{ each pair in parameters.matrix }}:
+ - GPU_Test_${{ pair.key }}
+ - Cleanup_Verification_${{ pair.key }}
+ variables:
+ UPLOAD_PATH: $(Build.SourcesDirectory)/log_dir
+ DOWNLOAD_PATH: $(Build.SourcesDirectory)/log_dir
+ ARTIFACT_NAME: "UT_coverage_report"
+ jobs:
+ - job: CollectDatafiles
+ steps:
+ - checkout: self
+
+ - task: DownloadPipelineArtifact@2
+ inputs:
+ patterns: 'ut-*_coverage/.coverage.*'
+ path: $(DOWNLOAD_PATH)
+
+ - task: UsePythonVersion@0
+ inputs:
+ versionSpec: '3.12'
+ displayName: 'Use Python 3.12'
+
+ - script: |
+ cd ${BUILD_SOURCESDIRECTORY}
+ pip install -U pip setuptools uv
+ uv pip install -r requirements.txt --extra-index-url https://download.pytorch.org/whl/cpu
+ uv pip install .
+ pip list
+ bash .azure-pipelines/scripts/ut/collect_log.sh
+ env:
+ PYTHONUNBUFFERED: '1'
+ UV_NO_PROGRESS: '1'
+ UV_SYSTEM_PYTHON: '1'
+ displayName: "Collect UT Coverage"
+
+ - task: PublishPipelineArtifact@1
+ condition: succeededOrFailed()
+ inputs:
+ targetPath: $(UPLOAD_PATH)/coverage_PR
+ artifact: $(ARTIFACT_NAME)
+ publishLocation: "pipeline"
+
+ - task: PublishCodeCoverageResults@2
+ inputs:
+ summaryFileLocation: $(Build.SourcesDirectory)/log_dir/coverage_PR/coverage.xml