Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Fix cuda ut workflow (#1472)

Sun, Xuehaob55ffb90264b
diff --git a/.azure-pipelines/scripts/ut/run_ut.sh b/.azure-pipelines/scripts/ut/run_ut.sh
index eb80adff3..fd536deb4 100644
--- a/.azure-pipelines/scripts/ut/run_ut.sh
+++ b/.azure-pipelines/scripts/ut/run_ut.sh
@@ -3,6 +3,7 @@ set -xe
test_part=$1
export TQDM_MININTERVAL=60
+export HF_HUB_DISABLE_PROGRESS_BARS=1
echo "##[group]set up UT env..."
uv pip install pytest-cov pytest-html
uv pip list
diff --git a/.azure-pipelines/scripts/ut/run_ut_cuda.sh b/.azure-pipelines/scripts/ut/run_ut_cuda.sh
index a9f621f4a..ba823a8bf 100644
--- a/.azure-pipelines/scripts/ut/run_ut_cuda.sh
+++ b/.azure-pipelines/scripts/ut/run_ut_cuda.sh
@@ -18,16 +18,16 @@ function create_conda_env() {
[[ -d ${HOME}/miniconda3/bin ]] && export PATH=${HOME}/miniconda3/bin/:$PATH
# create conda env
- source activate base
+ source activate base > /dev/null 2>&1
if conda info --envs | grep -q "^$CONDA_ENV_NAME\s"; then conda remove -n ${CONDA_ENV_NAME} --all -y; fi
conda create -n ${CONDA_ENV_NAME} python=${PYTHON_VERSION} setuptools -y
- source activate ${CONDA_ENV_NAME}
+ source activate ${CONDA_ENV_NAME} > /dev/null 2>&1
conda install -c conda-forge git gxx=11.2.0 gcc=11.2.0 gdb sysroot_linux-64 libgcc uv -y
export LD_PRELOAD=${CONDA_PREFIX}/lib/libstdc++.so.6
# install AutoRound
cd ${REPO_PATH}
- uv pip install torch==2.9.1 torchvision
+ uv pip install torch==2.10.0 torchvision
uv pip install -r requirements.txt
if [ -d "/proc/driver/nvidia" ]; then
export PATH=/usr/local/cuda/bin${PATH:+:${PATH}}
@@ -95,21 +95,21 @@ function run_unit_test() {
rm -rf .coverage* *.xml *.html
uv pip install -v git+https://github.com/casper-hansen/AutoAWQ.git --no-build-isolation
- uv pip install https://github.com/ModelCloud/GPTQModel/releases/download/v5.6.0/gptqmodel-5.6.0+cu126torch2.9-cp310-cp310-linux_x86_64.whl --no-build-isolation
+ uv pip install gptqmodel --no-build-isolation
uv pip install -r https://raw.githubusercontent.com/ModelCloud/GPTQModel/refs/heads/main/requirements.txt
CMAKE_ARGS="-DGGML_CUDA=on -DLLAVA_BUILD=off" uv pip install llama-cpp-python
uv pip install 'git+https://github.com/ggml-org/llama.cpp.git#subdirectory=gguf-py'
uv pip install -r test_cuda/requirements.txt
uv pip install -r test_cuda/requirements_diffusion.txt
- uv pip install -r test_cuda/requirements_sglang.txt
- uv pip install transformers==4.57.6
+ uv pip install torch==2.10.0 torchvision
+ uv pip install -U transformers
pip list > ${LOG_DIR}/ut_pip_list.txt
export COVERAGE_RCFILE=${REPO_PATH}/.azure-pipelines/scripts/ut/.coverage
local auto_round_path=$(python -c 'import auto_round; print(auto_round.__path__[0])')
# run unit tests individually with separate logs
- for test_file in $(find ./test_cuda -name "test_*.py" ! -name "test_*vlms.py" ! -name "test_llmc*.py" | sort); do
+ for test_file in $(find ./test_cuda -name "test_*.py" ! -name "test_*vlms.py" ! -name "test_llmc*.py" ! -name "test_*sglang*.py" | sort); do
local test_basename=$(basename ${test_file} .py)
local ut_log_name=${LOG_DIR}/unittest_cuda_${test_basename}.log
echo "Running ${test_file}..."
@@ -169,7 +169,6 @@ function run_unit_test_llmc() {
cd ${REPO_PATH}/test
rm -rf .coverage* *.xml *.html
-
uv pip install -r test_cuda/requirements_llmc.txt
pip list > ${LOG_DIR}/llmc_ut_pip_list.txt
@@ -193,9 +192,40 @@ function run_unit_test_llmc() {
fi
}
+function run_unit_test_sglang() {
+ # install unit test dependencies
+ create_conda_env
+
+ cd ${REPO_PATH}/test
+ rm -rf .coverage* *.xml *.html
+ uv pip install torch==2.9.1 torchvision
+ uv pip install -r test_cuda/requirements_sglang.txt
+
+ pip list > ${LOG_DIR}/sglang_ut_pip_list.txt
+ export COVERAGE_RCFILE=${REPO_PATH}/.azure-pipelines/scripts/ut/.coverage
+ local auto_round_path=$(python -c 'import auto_round; print(auto_round.__path__[0])')
+
+ # run unit tests individually with separate logs
+ for test_file in $(find ./test_cuda -name "test_sglang*.py" | sort); do
+ local test_basename=$(basename ${test_file} .py)
+ local ut_log_name=${LOG_DIR}/unittest_cuda_sglang_${test_basename}.log
+ echo "Running ${test_file}..."
+
+ python -m pytest --cov="${auto_round_path}" --cov-report term --html=report_sglang.html --self-contained-html --cov-report xml:coverage_sglang.xml --cov-append -vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}
+ done
+
+ mv report_sglang.html ${LOG_DIR}/
+ mv coverage_sglang.xml ${LOG_DIR}/
+ # Print test results table and check for failures
+ if ! print_test_results_table "unittest_cuda_sglang_test*.log" "CUDA SGLang Unit Tests"; then
+ echo "Some CUDA SGLang unit tests failed. Please check the individual log files for details."
+ fi
+}
+
function main() {
run_unit_test_vlm
run_unit_test_llmc
+ run_unit_test_sglang
run_unit_test
cat ${SUMMARY_LOG}
}
diff --git a/.azure-pipelines/scripts/ut/run_ut_hpu.sh b/.azure-pipelines/scripts/ut/run_ut_hpu.sh
index f90fdb7c0..c76588598 100644
--- a/.azure-pipelines/scripts/ut/run_ut_hpu.sh
+++ b/.azure-pipelines/scripts/ut/run_ut_hpu.sh
@@ -4,6 +4,7 @@ set -xe
# install requirements
echo "set up UT env..."
export TQDM_MININTERVAL=60
+export HF_HUB_DISABLE_PROGRESS_BARS=1
pip install pytest-cov pytest-html
pip list
diff --git a/.azure-pipelines/scripts/ut/run_ut_xpu.sh b/.azure-pipelines/scripts/ut/run_ut_xpu.sh
index c7e5da245..b8f7bb875 100644
--- a/.azure-pipelines/scripts/ut/run_ut_xpu.sh
+++ b/.azure-pipelines/scripts/ut/run_ut_xpu.sh
@@ -11,6 +11,7 @@ rm -rf /auto-round/auto_round
cd /auto-round/test || exit 1
export TQDM_MININTERVAL=60
+export HF_HUB_DISABLE_PROGRESS_BARS=1
export ZE_AFFINITY_MASK=2,3 # set xpu affinity
export LD_LIBRARY_PATH=${HOME}/.venv/lib/:$LD_LIBRARY_PATH
export COVERAGE_RCFILE=/auto-round/.azure-pipelines/scripts/ut/.coverage
diff --git a/.github/workflows/compatibility-test.yml b/.github/workflows/compatibility-test.yml
index 218368d4b..64981aa45 100644
--- a/.github/workflows/compatibility-test.yml
+++ b/.github/workflows/compatibility-test.yml
@@ -81,3 +81,4 @@ jobs:
env:
TQDM_MININTERVAL: "60"
PYTHONUNBUFFERED: "1"
+ HF_HUB_DISABLE_PROGRESS_BARS: "1"