Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Fix Cuda UT coverage collect (#2234)

chen, suyuee5693be884cd
diff --git a/.azure-pipelines/nightly-test-cuda.yml b/.azure-pipelines/nightly-test-cuda.yml
new file mode 100644
index 000000000..691beccc8
--- /dev/null
+++ b/.azure-pipelines/nightly-test-cuda.yml
@@ -0,0 +1,221 @@
+# Nightly CUDA pipeline: runs the slower CUDA integration + e2e suites
+# (sglang, llmc, vllm) that are excluded from the fast PR unit-test pipeline
+# (unit-test-cuda.yml). Scheduled daily; can also be triggered manually from
+# the Azure DevOps UI.
+
+trigger: none
+
+pr:
+ autoCancel: true
+ drafts: false
+ branches:
+ include:
+ - main
+ paths:
+ include:
+ - auto_round
+ - auto_round_extension
+ - test/integration/test_cuda
+ - setup.py
+ - requirements.txt
+ - .azure-pipelines/scripts/cuda_unit_test
+ - .azure-pipelines/scripts/ut/detect_changed_tests.sh
+ - .azure-pipelines/nightly-test-cuda.yml
+
+ exclude:
+ - "*.md"
+ - "**/*.md"
+ - auto_round_extension/ark
+
+schedules:
+ - cron: "0 18 * * *" # 02:00 Asia/Shanghai every day
+ displayName: Daily CUDA nightly run
+ branches:
+ include:
+ - main
+ always: true
+
+variables:
+ - name: POOL_NAME
+ value: "RunPod-GPU-Pool"
+ - name: AGENT_NAME
+ value: "runpod-agent-$(Build.BuildId)"
+
+stages:
+ - stage: Start_Pod
+ displayName: "Start Pod"
+ dependsOn: []
+ jobs:
+ - job: Start_Pod_Job
+ displayName: "Launch Pod & Run CUDA Nightly Tests"
+ timeoutInMinutes: 60
+ pool:
+ vmImage: "ubuntu-latest"
+ steps:
+ - checkout: self
+
+ - task: UsePythonVersion@0
+ inputs:
+ versionSpec: "3.13"
+ displayName: "Use Python 3.13"
+
+ - task: Bash@3
+ displayName: "Install dependencies"
+ inputs:
+ targetType: "inline"
+ script: |
+ pip install requests
+
+ - task: Bash@3
+ displayName: "Create RunPod Instance"
+ name: create_runpod
+ inputs:
+ targetType: "inline"
+ script: |
+ CUDA_VERSION="13.0"
+
+ python .azure-pipelines/scripts/cuda_unit_test/runpod_manager.py \
+ --action create \
+ --api_key $(RUNPOD_API_KEY) \
+ --name "nightly-$(Build.BuildId)" \
+ --container_disk_size "100" \
+ --part 0 \
+ --cuda_version "${CUDA_VERSION}" \
+ --env AZP_URL=$(System.CollectionUri) AZP_TOKEN=$(AZP_DEVOPS_PAT) AZP_POOL=$(POOL_NAME) AZP_AGENT_NAME="$(AGENT_NAME)"
+ env:
+ PYTHONUNBUFFERED: '1'
+
+ - task: Bash@3
+ displayName: "Wait for Pod to be Online"
+ name: wait_for_pod
+ inputs:
+ targetType: "inline"
+ script: |
+ python .azure-pipelines/scripts/cuda_unit_test/runpod_manager.py \
+ --action wait \
+ --api_key $(RUNPOD_API_KEY) \
+ --name "nightly-$(Build.BuildId)"
+ env:
+ PYTHONUNBUFFERED: '1'
+
+ - task: Bash@3
+ displayName: "Wait for Agent to be Online"
+ name: wait_for_agent
+ inputs:
+ targetType: "inline"
+ script: |
+ python .azure-pipelines/scripts/cuda_unit_test/azure_agent.py \
+ --action wait \
+ --url "$(System.CollectionUri)" \
+ --pat "$(AZP_DEVOPS_PAT)" \
+ --pool "$(POOL_NAME)" \
+ --agent "$(AGENT_NAME)"
+ env:
+ PYTHONUNBUFFERED: '1'
+
+ - stage: GPU_Test
+ displayName: "GPU Nightly Test"
+ dependsOn: Start_Pod
+ jobs:
+ - job: GpuTests
+ timeoutInMinutes: 300
+ pool:
+ name: "$(POOL_NAME)"
+ demands: Agent.Name -equals runpod-agent-$(Build.BuildId)
+ steps:
+ - checkout: self
+ clean: true
+ displayName: "Checkout Repo"
+
+ - script: |
+ echo "Running on GPU Agent: $(Agent.Name)"
+ nvidia-smi
+ lscpu
+ cd .azure-pipelines/scripts/cuda_unit_test
+ nohup uv run monitor_gpu.py daemon > monitor.log 2>&1 &
+ displayName: "Verify GPU Access"
+
+ - script: |
+ echo "##[group]Installing Python dependencies..."
+ uv pip install -U "pytest-cov" "requests" "huggingface_hub"
+ hf auth login --token "$(HF_TOKEN)"
+ echo "##[endgroup]"
+ displayName: "Install Python Dependencies"
+
+ - script: |
+ export UV_NO_CACHE=0
+ export FILTER_CHANGED_TESTS=true
+ bash .azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh \
+ --test_case=nightly \
+ --test_part=0
+ displayName: "Run GPU Nightly Tests"
+
+ - task: PublishPipelineArtifact@1
+ condition: succeeded()
+ inputs:
+ targetPath: $(Build.SourcesDirectory)/log_dir
+ artifact: "ut-nightly_coverage"
+ publishLocation: "pipeline"
+ displayName: "Publish UT coverage artifact"
+
+ - script: |
+ cd .azure-pipelines/scripts/cuda_unit_test
+ uv run monitor_gpu.py stop
+ condition: always()
+ displayName: "GPU Monitor"
+
+ - stage: Cleanup_Verification
+ displayName: "Verify Pod Cleanup"
+ dependsOn:
+ - Start_Pod
+ - GPU_Test
+ condition: always()
+ jobs:
+ - job: Verify_Cleanup
+ pool:
+ vmImage: "ubuntu-latest"
+ steps:
+ - checkout: self
+
+ - task: UsePythonVersion@0
+ inputs:
+ versionSpec: "3.13"
+ displayName: "Use Python 3.13"
+
+ - task: Bash@3
+ displayName: "Install dependencies"
+ condition: always()
+ inputs:
+ targetType: "inline"
+ script: |
+ pip install requests
+
+ - task: Bash@3
+ displayName: "Terminate RunPod Instance"
+ condition: always()
+ inputs:
+ targetType: "inline"
+ script: |
+ echo "Terminating Pod: nightly-$(Build.BuildId)"
+ python .azure-pipelines/scripts/cuda_unit_test/runpod_manager.py \
+ --action terminate \
+ --api_key $(RUNPOD_API_KEY) \
+ --name "nightly-$(Build.BuildId)"
+ env:
+ PYTHONUNBUFFERED: '1'
+
+ - task: Bash@3
+ displayName: "Deregister Agent from Pool"
+ condition: always()
+ inputs:
+ targetType: "inline"
+ script: |
+ cd ${BUILD_SOURCESDIRECTORY}
+ python .azure-pipelines/scripts/cuda_unit_test/azure_agent.py \
+ --action deregister \
+ --url "$(System.CollectionUri)" \
+ --pat "$(AZP_DEVOPS_PAT)" \
+ --pool "$(POOL_NAME)" \
+ --agent "$(AGENT_NAME)"
+ env:
+ PYTHONUNBUFFERED: '1'
diff --git a/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh b/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh
index 1a65864b8..2e08c6a9e 100644
--- a/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh
+++ b/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh
@@ -36,6 +36,7 @@ function setup_environment() {
export TQDM_MININTERVAL=120
export CUDA_VISIBLE_DEVICES=0
export HF_HUB_DISABLE_PROGRESS_BARS=1
+ export COVERAGE_RCFILE="${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/.coveragerc"
}
function print_summary() {
@@ -71,8 +72,6 @@ function setup_basic_test_env() {
echo "##[endgroup]"
uv pip list
- export COVERAGE_RCFILE="${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/.coveragerc"
-
cd "${BUILD_SOURCESDIRECTORY}/test" || exit 1
}
@@ -86,7 +85,7 @@ function run_common_group() {
if [ -n "${group_tests}" ]; then
echo "##[group]Running common tests (${group_name})..."
- local ut_log_name="${LOG_DIR}/unittest_common_${group_name}.log"
+ local ut_log_name="${LOG_DIR}/unittest_cuda_common_${group_name}.log"
pytest -m "not skip_ci" \
--cov=auto_round --cov-report= --cov-append \
-vs --junitxml="${ut_log_name%.log}.xml" \
@@ -108,8 +107,6 @@ function run_common_unit_test() {
run_common_group "${group_name}" "${entry}"
fi
done
-
- python ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/collect_result.py --test-type "Common Unit Tests" --log-pattern "unittest_common_*.log" --log-dir ${LOG_DIR} --summary-log ${SUMMARY_LOG}
}
@@ -146,9 +143,6 @@ function run_unit_test() {
${test_file} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
done
- [ -f .coverage ] && cp .coverage "${LOG_DIR}/.coverage.part${test_part}"
-
- python ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/collect_result.py --test-type "CUDA Unit Tests" --log-pattern "unittest_cuda_test_*.log" --log-dir ${LOG_DIR} --summary-log ${SUMMARY_LOG}
}
function run_unit_test_llmc() {
@@ -168,8 +162,6 @@ function run_unit_test_llmc() {
cd "${BUILD_SOURCESDIRECTORY}/test" || exit 1
- export COVERAGE_RCFILE="${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/.coveragerc"
-
for test_file in $(find ./integration/test_cuda -name "test_llmc*.py" | sort); do
echo "##[group]Running ${test_file}..."
local test_basename=$(basename ${test_file} .py)
@@ -180,9 +172,6 @@ function run_unit_test_llmc() {
${test_file} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
done
- [ -f .coverage ] && cp .coverage "${LOG_DIR}/.coverage.llmc"
-
- python ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/collect_result.py --test-type "CUDA LLMC Tests" --log-pattern "unittest_cuda_llmc_test_*.log" --log-dir ${LOG_DIR} --summary-log ${SUMMARY_LOG}
}
function run_unit_test_sglang() {
@@ -202,7 +191,6 @@ function run_unit_test_sglang() {
echo "##[endgroup]"
cd "${BUILD_SOURCESDIRECTORY}/test" || exit 1
- export COVERAGE_RCFILE="${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/.coveragerc"
for test_file in $(find ./integration/test_cuda ./e2e/test_cuda -name "test_sglang*.py" | sort); do
echo "##[group]Running ${test_file}..."
@@ -214,9 +202,6 @@ function run_unit_test_sglang() {
${test_file} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
done
- [ -f .coverage ] && cp .coverage "${LOG_DIR}/.coverage.sglang"
-
- python ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/collect_result.py --test-type "CUDA SGLang Tests" --log-pattern "unittest_cuda_sglang_test_*.log" --log-dir ${LOG_DIR} --summary-log ${SUMMARY_LOG}
}
function run_unit_test_vllm() {
@@ -236,7 +221,6 @@ function run_unit_test_vllm() {
echo "##[endgroup]"
cd "${BUILD_SOURCESDIRECTORY}/test" || exit 1
- export COVERAGE_RCFILE="${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/.coveragerc"
for test_file in $(find ./integration/test_cuda ./e2e/test_cuda -name "test_vllm*.py" | sort); do
echo "##[group]Running ${test_file}..."
@@ -248,9 +232,16 @@ function run_unit_test_vllm() {
${test_file} 2>&1 | tee ${ut_log_name}
echo "##[endgroup]"
done
- [ -f .coverage ] && cp .coverage "${LOG_DIR}/.coverage.vllm"
+}
+
+function collect_log() {
+ touch "${SUMMARY_LOG}"
+ python ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/collect_result.py \
+ --test-type "Unit Tests" --log-pattern "unittest_cuda_*.log" --log-dir ${LOG_DIR} --summary-log ${SUMMARY_LOG}
- python ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/collect_result.py --test-type "CUDA VLLM Tests" --log-pattern "unittest_cuda_vllm_test_*.log" --log-dir ${LOG_DIR} --summary-log ${SUMMARY_LOG}
+ if [ -f .coverage ]; then
+ cp .coverage "${LOG_DIR}/.coverage.part${test_part}"
+ fi
}
function main() {
@@ -274,6 +265,7 @@ function main() {
echo "##[error]Invalid test case specified: ${test_case}. Please use 'nightly' or 'ci'."
exit 1
fi
+ collect_log
check_storage_usage
print_summary
}
diff --git a/.azure-pipelines/scripts/ut/run_ut_cuda.sh b/.azure-pipelines/scripts/ut/run_ut_cuda.sh
index dccfee9ba..650f87b6c 100644
--- a/.azure-pipelines/scripts/ut/run_ut_cuda.sh
+++ b/.azure-pipelines/scripts/ut/run_ut_cuda.sh
@@ -1,4 +1,6 @@
#!/bin/bash
+# This script used for weekly unit test on CUDA environment, and will not be triggered by PR or CI pipeline.
+# CI related cuda UT could be found in .azure-pipelines/scripts/cuda_unit_test.
set -xe
CONDA_ENV_NAME="unittest_cuda"
diff --git a/test/README.md b/test/README.md
index f9dddd78c..895fdb5bb 100644
--- a/test/README.md
+++ b/test/README.md
@@ -18,7 +18,7 @@ Tests are split into three **tiers**, then by hardware backend
- **`unit/`** — fast, self-contained tests. Run on **every PR** (`unit-test*.yml`).
- **`integration/`** — tests against third-party frameworks (vLLM, SGLang,
LLMCompressor, INC, HuggingFace). Run **nightly** — CPU via
- `nightly-test-cpu.yml`, XPU via `nightly-test-xpu.yml`.
+ `nightly-test-cpu.yml`, XPU via `nightly-test-xpu.yml`, CUDA via `nightly-test-cuda.yml`.
- **`e2e/`** — full-model / real inference-engine tests. Run **manually**
Within `unit/test_cpu/` and `unit/test_cuda/`, tests are grouped by functionality:
diff --git a/test/integration/README.md b/test/integration/README.md
index 100e30d67..68a16680e 100644
--- a/test/integration/README.md
+++ b/test/integration/README.md
@@ -10,6 +10,7 @@ pytest test/integration/ -v
# Run specific integration tests
pytest test/integration/test_cpu/ -v
+pytest test/integration/test_xpu/ -v
pytest test/integration/test_cuda/ -v
```
@@ -19,3 +20,4 @@ These tests run in the **nightly** CI pipelines:
- CPU integration → `.azure-pipelines/nightly-test.yml`
- XPU integration → `.azure-pipelines/nightly-test-xpu.yml`
+- CUDA integration → `.azure-pipelines/nightly-test-cuda.yml`