PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Fix Cuda UT coverage collect (#2234)
chen, suyuee5693be884cd
diff --git a/.azure-pipelines/nightly-test-cuda.yml b/.azure-pipelines/nightly-test-cuda.ymlnew file mode 100644index 000000000..691beccc8--- /dev/null+++ b/.azure-pipelines/nightly-test-cuda.yml@@ -0,0 +1,221 @@+# Nightly CUDA pipeline: runs the slower CUDA integration + e2e suites+# (sglang, llmc, vllm) that are excluded from the fast PR unit-test pipeline+# (unit-test-cuda.yml). Scheduled daily; can also be triggered manually from+# the Azure DevOps UI.++trigger: none++pr:+ autoCancel: true+ drafts: false+ branches:+ include:+ - main+ paths:+ include:+ - auto_round+ - auto_round_extension+ - test/integration/test_cuda+ - setup.py+ - requirements.txt+ - .azure-pipelines/scripts/cuda_unit_test+ - .azure-pipelines/scripts/ut/detect_changed_tests.sh+ - .azure-pipelines/nightly-test-cuda.yml++ exclude:+ - "*.md"+ - "**/*.md"+ - auto_round_extension/ark++schedules:+ - cron: "0 18 * * *" # 02:00 Asia/Shanghai every day+ displayName: Daily CUDA nightly run+ branches:+ include:+ - main+ always: true++variables:+ - name: POOL_NAME+ value: "RunPod-GPU-Pool"+ - name: AGENT_NAME+ value: "runpod-agent-$(Build.BuildId)"++stages:+ - stage: Start_Pod+ displayName: "Start Pod"+ dependsOn: []+ jobs:+ - job: Start_Pod_Job+ displayName: "Launch Pod & Run CUDA Nightly Tests"+ timeoutInMinutes: 60+ pool:+ vmImage: "ubuntu-latest"+ steps:+ - checkout: self++ - task: UsePythonVersion@0+ inputs:+ versionSpec: "3.13"+ displayName: "Use Python 3.13"++ - task: Bash@3+ displayName: "Install dependencies"+ inputs:+ targetType: "inline"+ script: |+ pip install requests++ - task: Bash@3+ displayName: "Create RunPod Instance"+ name: create_runpod+ inputs:+ targetType: "inline"+ script: |+ CUDA_VERSION="13.0"++ python .azure-pipelines/scripts/cuda_unit_test/runpod_manager.py \+ --action create \+ --api_key $(RUNPOD_API_KEY) \+ --name "nightly-$(Build.BuildId)" \+ --container_disk_size "100" \+ --part 0 \+ --cuda_version "${CUDA_VERSION}" \+ --env AZP_URL=$(System.CollectionUri) AZP_TOKEN=$(AZP_DEVOPS_PAT) AZP_POOL=$(POOL_NAME) AZP_AGENT_NAME="$(AGENT_NAME)"+ env:+ PYTHONUNBUFFERED: '1'++ - task: Bash@3+ displayName: "Wait for Pod to be Online"+ name: wait_for_pod+ inputs:+ targetType: "inline"+ script: |+ python .azure-pipelines/scripts/cuda_unit_test/runpod_manager.py \+ --action wait \+ --api_key $(RUNPOD_API_KEY) \+ --name "nightly-$(Build.BuildId)"+ env:+ PYTHONUNBUFFERED: '1'++ - task: Bash@3+ displayName: "Wait for Agent to be Online"+ name: wait_for_agent+ inputs:+ targetType: "inline"+ script: |+ python .azure-pipelines/scripts/cuda_unit_test/azure_agent.py \+ --action wait \+ --url "$(System.CollectionUri)" \+ --pat "$(AZP_DEVOPS_PAT)" \+ --pool "$(POOL_NAME)" \+ --agent "$(AGENT_NAME)"+ env:+ PYTHONUNBUFFERED: '1'++ - stage: GPU_Test+ displayName: "GPU Nightly Test"+ dependsOn: Start_Pod+ jobs:+ - job: GpuTests+ timeoutInMinutes: 300+ pool:+ name: "$(POOL_NAME)"+ demands: Agent.Name -equals runpod-agent-$(Build.BuildId)+ steps:+ - checkout: self+ clean: true+ displayName: "Checkout Repo"++ - script: |+ echo "Running on GPU Agent: $(Agent.Name)"+ nvidia-smi+ lscpu+ cd .azure-pipelines/scripts/cuda_unit_test+ nohup uv run monitor_gpu.py daemon > monitor.log 2>&1 &+ displayName: "Verify GPU Access"++ - script: |+ echo "##[group]Installing Python dependencies..."+ uv pip install -U "pytest-cov" "requests" "huggingface_hub"+ hf auth login --token "$(HF_TOKEN)"+ echo "##[endgroup]"+ displayName: "Install Python Dependencies"++ - script: |+ export UV_NO_CACHE=0+ export FILTER_CHANGED_TESTS=true+ bash .azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh \+ --test_case=nightly \+ --test_part=0+ displayName: "Run GPU Nightly Tests"++ - task: PublishPipelineArtifact@1+ condition: succeeded()+ inputs:+ targetPath: $(Build.SourcesDirectory)/log_dir+ artifact: "ut-nightly_coverage"+ publishLocation: "pipeline"+ displayName: "Publish UT coverage artifact"++ - script: |+ cd .azure-pipelines/scripts/cuda_unit_test+ uv run monitor_gpu.py stop+ condition: always()+ displayName: "GPU Monitor"++ - stage: Cleanup_Verification+ displayName: "Verify Pod Cleanup"+ dependsOn:+ - Start_Pod+ - GPU_Test+ condition: always()+ jobs:+ - job: Verify_Cleanup+ pool:+ vmImage: "ubuntu-latest"+ steps:+ - checkout: self++ - task: UsePythonVersion@0+ inputs:+ versionSpec: "3.13"+ displayName: "Use Python 3.13"++ - task: Bash@3+ displayName: "Install dependencies"+ condition: always()+ inputs:+ targetType: "inline"+ script: |+ pip install requests++ - task: Bash@3+ displayName: "Terminate RunPod Instance"+ condition: always()+ inputs:+ targetType: "inline"+ script: |+ echo "Terminating Pod: nightly-$(Build.BuildId)"+ python .azure-pipelines/scripts/cuda_unit_test/runpod_manager.py \+ --action terminate \+ --api_key $(RUNPOD_API_KEY) \+ --name "nightly-$(Build.BuildId)"+ env:+ PYTHONUNBUFFERED: '1'++ - task: Bash@3+ displayName: "Deregister Agent from Pool"+ condition: always()+ inputs:+ targetType: "inline"+ script: |+ cd ${BUILD_SOURCESDIRECTORY}+ python .azure-pipelines/scripts/cuda_unit_test/azure_agent.py \+ --action deregister \+ --url "$(System.CollectionUri)" \+ --pat "$(AZP_DEVOPS_PAT)" \+ --pool "$(POOL_NAME)" \+ --agent "$(AGENT_NAME)"+ env:+ PYTHONUNBUFFERED: '1'diff --git a/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh b/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.shindex 1a65864b8..2e08c6a9e 100644--- a/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh+++ b/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh@@ -36,6 +36,7 @@ function setup_environment() {export TQDM_MININTERVAL=120export CUDA_VISIBLE_DEVICES=0export HF_HUB_DISABLE_PROGRESS_BARS=1+ export COVERAGE_RCFILE="${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/.coveragerc"}function print_summary() {@@ -71,8 +72,6 @@ function setup_basic_test_env() {echo "##[endgroup]"uv pip list- export COVERAGE_RCFILE="${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/.coveragerc"-cd "${BUILD_SOURCESDIRECTORY}/test" || exit 1}@@ -86,7 +85,7 @@ function run_common_group() {if [ -n "${group_tests}" ]; thenecho "##[group]Running common tests (${group_name})..."- local ut_log_name="${LOG_DIR}/unittest_common_${group_name}.log"+ local ut_log_name="${LOG_DIR}/unittest_cuda_common_${group_name}.log"pytest -m "not skip_ci" \--cov=auto_round --cov-report= --cov-append \-vs --junitxml="${ut_log_name%.log}.xml" \@@ -108,8 +107,6 @@ function run_common_unit_test() {run_common_group "${group_name}" "${entry}"fidone-- python ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/collect_result.py --test-type "Common Unit Tests" --log-pattern "unittest_common_*.log" --log-dir ${LOG_DIR} --summary-log ${SUMMARY_LOG}}@@ -146,9 +143,6 @@ function run_unit_test() {${test_file} 2>&1 | tee ${ut_log_name}echo "##[endgroup]"done- [ -f .coverage ] && cp .coverage "${LOG_DIR}/.coverage.part${test_part}"-- python ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/collect_result.py --test-type "CUDA Unit Tests" --log-pattern "unittest_cuda_test_*.log" --log-dir ${LOG_DIR} --summary-log ${SUMMARY_LOG}}function run_unit_test_llmc() {@@ -168,8 +162,6 @@ function run_unit_test_llmc() {cd "${BUILD_SOURCESDIRECTORY}/test" || exit 1- export COVERAGE_RCFILE="${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/.coveragerc"-for test_file in $(find ./integration/test_cuda -name "test_llmc*.py" | sort); doecho "##[group]Running ${test_file}..."local test_basename=$(basename ${test_file} .py)@@ -180,9 +172,6 @@ function run_unit_test_llmc() {${test_file} 2>&1 | tee ${ut_log_name}echo "##[endgroup]"done- [ -f .coverage ] && cp .coverage "${LOG_DIR}/.coverage.llmc"-- python ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/collect_result.py --test-type "CUDA LLMC Tests" --log-pattern "unittest_cuda_llmc_test_*.log" --log-dir ${LOG_DIR} --summary-log ${SUMMARY_LOG}}function run_unit_test_sglang() {@@ -202,7 +191,6 @@ function run_unit_test_sglang() {echo "##[endgroup]"cd "${BUILD_SOURCESDIRECTORY}/test" || exit 1- export COVERAGE_RCFILE="${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/.coveragerc"for test_file in $(find ./integration/test_cuda ./e2e/test_cuda -name "test_sglang*.py" | sort); doecho "##[group]Running ${test_file}..."@@ -214,9 +202,6 @@ function run_unit_test_sglang() {${test_file} 2>&1 | tee ${ut_log_name}echo "##[endgroup]"done- [ -f .coverage ] && cp .coverage "${LOG_DIR}/.coverage.sglang"-- python ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/collect_result.py --test-type "CUDA SGLang Tests" --log-pattern "unittest_cuda_sglang_test_*.log" --log-dir ${LOG_DIR} --summary-log ${SUMMARY_LOG}}function run_unit_test_vllm() {@@ -236,7 +221,6 @@ function run_unit_test_vllm() {echo "##[endgroup]"cd "${BUILD_SOURCESDIRECTORY}/test" || exit 1- export COVERAGE_RCFILE="${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/.coveragerc"for test_file in $(find ./integration/test_cuda ./e2e/test_cuda -name "test_vllm*.py" | sort); doecho "##[group]Running ${test_file}..."@@ -248,9 +232,16 @@ function run_unit_test_vllm() {${test_file} 2>&1 | tee ${ut_log_name}echo "##[endgroup]"done- [ -f .coverage ] && cp .coverage "${LOG_DIR}/.coverage.vllm"+}++function collect_log() {+ touch "${SUMMARY_LOG}"+ python ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/collect_result.py \+ --test-type "Unit Tests" --log-pattern "unittest_cuda_*.log" --log-dir ${LOG_DIR} --summary-log ${SUMMARY_LOG}- python ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/collect_result.py --test-type "CUDA VLLM Tests" --log-pattern "unittest_cuda_vllm_test_*.log" --log-dir ${LOG_DIR} --summary-log ${SUMMARY_LOG}+ if [ -f .coverage ]; then+ cp .coverage "${LOG_DIR}/.coverage.part${test_part}"+ fi}function main() {@@ -274,6 +265,7 @@ function main() {echo "##[error]Invalid test case specified: ${test_case}. Please use 'nightly' or 'ci'."exit 1fi+ collect_logcheck_storage_usageprint_summary}diff --git a/.azure-pipelines/scripts/ut/run_ut_cuda.sh b/.azure-pipelines/scripts/ut/run_ut_cuda.shindex dccfee9ba..650f87b6c 100644--- a/.azure-pipelines/scripts/ut/run_ut_cuda.sh+++ b/.azure-pipelines/scripts/ut/run_ut_cuda.sh@@ -1,4 +1,6 @@#!/bin/bash+# This script used for weekly unit test on CUDA environment, and will not be triggered by PR or CI pipeline.+# CI related cuda UT could be found in .azure-pipelines/scripts/cuda_unit_test.set -xeCONDA_ENV_NAME="unittest_cuda"diff --git a/test/README.md b/test/README.mdindex f9dddd78c..895fdb5bb 100644--- a/test/README.md+++ b/test/README.md@@ -18,7 +18,7 @@ Tests are split into three **tiers**, then by hardware backend- **`unit/`** — fast, self-contained tests. Run on **every PR** (`unit-test*.yml`).- **`integration/`** — tests against third-party frameworks (vLLM, SGLang,LLMCompressor, INC, HuggingFace). Run **nightly** — CPU via- `nightly-test-cpu.yml`, XPU via `nightly-test-xpu.yml`.+ `nightly-test-cpu.yml`, XPU via `nightly-test-xpu.yml`, CUDA via `nightly-test-cuda.yml`.- **`e2e/`** — full-model / real inference-engine tests. Run **manually**Within `unit/test_cpu/` and `unit/test_cuda/`, tests are grouped by functionality:diff --git a/test/integration/README.md b/test/integration/README.mdindex 100e30d67..68a16680e 100644--- a/test/integration/README.md+++ b/test/integration/README.md@@ -10,6 +10,7 @@ pytest test/integration/ -v# Run specific integration testspytest test/integration/test_cpu/ -v+pytest test/integration/test_xpu/ -vpytest test/integration/test_cuda/ -v```@@ -19,3 +20,4 @@ These tests run in the **nightly** CI pipelines:- CPU integration → `.azure-pipelines/nightly-test.yml`- XPU integration → `.azure-pipelines/nightly-test-xpu.yml`+- CUDA integration → `.azure-pipelines/nightly-test-cuda.yml`