Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Refine CI (#1644)

Sun, Xuehao9f54985cd71f
diff --git a/.azure-pipelines/scripts/cuda_unit_test/runpod_manager.py b/.azure-pipelines/scripts/cuda_unit_test/runpod_manager.py
index fcdfc8ab0..26faa1868 100644
--- a/.azure-pipelines/scripts/cuda_unit_test/runpod_manager.py
+++ b/.azure-pipelines/scripts/cuda_unit_test/runpod_manager.py
@@ -7,7 +7,6 @@ import requests
TARGET_GPUS = [
"NVIDIA GeForce RTX 4090",
- "NVIDIA RTX PRO 4500 Blackwell",
"NVIDIA GeForce RTX 5090",
"NVIDIA RTX 6000 Ada Generation",
"NVIDIA L40S",
@@ -47,90 +46,6 @@ REQUIRED_COUNT = 1
IMAGES_NAME = "xuehaosu/azure-agent:v0.1"
-def check_gpu_count(token):
- url = "https://api.runpod.io/graphql"
- ids_string = ", ".join([f'"{gid}"' for gid in TARGET_GPUS])
- graphql_query = """
- query GpuAvailability($input: GpuLowestPriceInput!) {
- gpuTypes(input: {ids: [%s]}) {
- id
- displayName
- memoryInGb
- secureCloud
- communityCloud
- maxGpuCount
- maxGpuCountSecureCloud
- maxGpuCountCommunityCloud
- minPodGpuCount
- lowestPrice(input: $input) {
- gpuName
- stockStatus
- minimumBidPrice
- uninterruptablePrice
- maxGpuCount
- maxUnreservedGpuCount
- availableGpuCounts
- rentedCount
- totalCount
- }
- }
- }
- """ % ids_string
-
- datacenter_id_string = ",".join(DATA_CENTER_SELECT_LIST)
- variables = {
- "input": {
- "gpuCount": 1,
- "secureCloud": True,
- "minMemoryInGb": 0,
- "minVcpuCount": 0,
- "dataCenterId": datacenter_id_string,
- }
- }
-
- try:
- response = requests.post(
- url,
- json={
- "query": graphql_query,
- "variables": variables,
- },
- headers={
- "Content-Type": "application/json",
- "Authorization": f"Bearer {token}",
- },
- )
- response.raise_for_status()
- data = response.json()
- all_gpus = data.get("data", {}).get("gpuTypes", [])
-
- id_to_gpu = {gpu["id"]: gpu for gpu in all_gpus}
- all_gpus = [id_to_gpu[gpu_id] for gpu_id in TARGET_GPUS if gpu_id in id_to_gpu]
-
- print("--- Checking target graphics card inventory ---\n")
-
- for gpu in all_gpus:
- gpu_id = gpu.get("id")
-
- if gpu_id in TARGET_GPUS:
- max_count = gpu.get("lowestPrice", {}).get("maxUnreservedGpuCount", 0)
-
- if REQUIRED_COUNT > max_count:
- continue
- else:
- return gpu_id
-
- print("❌ No compliant GPU found.")
- return None
-
- except requests.exceptions.RequestException as e:
- print(f"HTTP Error: {e}")
- sys.exit(1)
- except Exception as e:
- print(f"Error: {e}")
- sys.exit(1)
-
-
def run_create_pod(api_key, payload):
url = "https://rest.runpod.io/v1/pods"
headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
@@ -141,17 +56,29 @@ def run_create_pod(api_key, payload):
if response.status_code >= 500 or response.status_code == 400:
if attempt < max_retries:
- print(f"⚠️ {response.status_code} Error, Retrying in 60 seconds ({attempt + 1}/{max_retries})...")
- time.sleep(60)
+ print(f"⚠️ {response.status_code} Error, Retrying in 90 seconds ({attempt + 1}/{max_retries})...")
+ time.sleep(90)
continue
else:
print(f"❌ {response.status_code} Error, Reached maximum retry attempts ({max_retries}), giving up.")
- response.raise_for_status()
- result = response.json()
+ try:
+ result = response.json()
+ except ValueError:
+ result = {}
+
if "errors" in result:
- print("❌ Errors:")
+ print("❌ API Errors:")
print(json.dumps(result["errors"], indent=2))
+
+ try:
+ response.raise_for_status()
+ except requests.exceptions.HTTPError as e:
+ print(f"❌ HTTP Error: {e}")
+ if result:
+ print("Response payload:", json.dumps(result, indent=2))
+ else:
+ print("Response text:", response.text)
sys.exit(1)
return result
@@ -161,19 +88,6 @@ def create_pod(args):
if args.env:
env_dict = {kv.split("=", 1)[0]: kv.split("=", 1)[1] for kv in args.env}
- gpu_type = None
- for _ in range(10): # Try up to 10 times to find an available GPU
- gpu_type = check_gpu_count(args.api_key)
- if gpu_type:
- break
- else:
- print("⏳ No compliant GPU available. Retrying in 5 minutes...")
- time.sleep(60 * 5)
-
- if not gpu_type:
- print("❌ No compliant GPU found after multiple attempts. Exiting.")
- sys.exit(1)
-
payload = {
"cloudType": "SECURE",
"containerDiskInGb": args.container_disk_size,
@@ -181,7 +95,8 @@ def create_pod(args):
"dataCenterPriority": "availability",
"env": env_dict,
"gpuCount": args.gpu_count,
- "gpuTypeIds": [gpu_type],
+ "gpuTypeIds": TARGET_GPUS,
+ "gpuTypePriority": "custom",
"name": args.name,
"volumeInGb": 0,
"imageName": IMAGES_NAME,
diff --git a/.azure-pipelines/scripts/ut/run_ut_hpu.sh b/.azure-pipelines/scripts/ut/run_ut_hpu.sh
index 7a49c02e8..e8b36c1d3 100644
--- a/.azure-pipelines/scripts/ut/run_ut_hpu.sh
+++ b/.azure-pipelines/scripts/ut/run_ut_hpu.sh
@@ -1,44 +1,84 @@
#!/bin/bash
-set -xe
+set -e
+source /auto-round/.azure-pipelines/scripts/change_color.sh
-# install requirements
-echo "set up UT env..."
-export TQDM_MININTERVAL=60
-export HF_HUB_DISABLE_PROGRESS_BARS=1
-pip install pytest-cov pytest-html
-pip list
+function setup_environment() {
+ # install requirements
+ echo "##[group]set up UT env..."
+ export TZ='Asia/Shanghai'
+ export TQDM_MININTERVAL=60
+ export HF_HUB_DISABLE_PROGRESS_BARS=1
+ pip install pytest-cov pytest-html
+ pip list
+ echo "##[endgroup]"
-rm -rf /auto-round/auto_round
-cd /auto-round/test || exit 1
+ rm -rf /auto-round/auto_round
+ cd /auto-round/test || exit 1
-export LD_LIBRARY_PATH=/usr/local/lib/:$LD_LIBRARY_PATH
-export FORCE_BF16=1
-export COVERAGE_RCFILE=/auto-round/.azure-pipelines/scripts/ut/.coverage
-auto_round_path=$(python -c 'import auto_round; print(auto_round.__path__[0])')
+ export LD_LIBRARY_PATH=/usr/local/lib/:$LD_LIBRARY_PATH
+ export FORCE_BF16=1
+ export COVERAGE_RCFILE=/auto-round/.azure-pipelines/scripts/ut/.coverage
-LOG_DIR=/auto-round/log_dir
-mkdir -p ${LOG_DIR}
-ut_log_name=${LOG_DIR}/ut.log
+ LOG_DIR=/auto-round/log_dir
+ mkdir -p ${LOG_DIR}
+ ut_log_name=${LOG_DIR}/ut.log
+ SUMMARY_LOG="${LOG_DIR}/results_summary.log"
+}
-find ./test_hpu -name "test*.py" | sed "s,\.\/, PT_HPU_LAZY_MODE=1 python -m pytest --cov=\"${auto_round_path}\" --cov-report term --html=report.html --self-contained-html --cov-report xml:coverage.xml --cov-append -vs --disable-warnings ,g" > run_lazy.sh
-find ./test_hpu -name "test*.py" | sed "s,\.\/, PT_HPU_LAZY_MODE=0 python -m pytest --mode compile --cov=\"${auto_round_path}\" --cov-report term --html=report.html --self-contained-html --cov-report xml:coverage.xml --cov-append -vs --disable-warnings ,g" > run_compile.sh
+function run_unit_test() {
+ auto_round_path=$(python -c 'import auto_round; print(auto_round.__path__[0])')
-cat run_lazy.sh
-bash run_lazy.sh 2>&1 | tee ${ut_log_name}
+ for test_file in $(find ./test_hpu -name "test*.py" | sort); do
+ local test_basename=$(basename ${test_file} .py)
-cat run_compile.sh
-bash run_compile.sh 2>&1 | tee -a ${ut_log_name}
+ echo "##[group]Running ${test_file} in HPU lazy mode..."
+ local ut_log_name="${LOG_DIR}/unittest_lazy_${test_basename}.log"
+ PT_HPU_LAZY_MODE=1 pytest --cov="${auto_round_path}" \
+ --cov-report term --html=report.html --self-contained-html \
+ --cov-report xml:coverage.xml --cov-append -vs --disable-warnings \
+ ${test_file} 2>&1 | tee ${ut_log_name}
+ echo "##[endgroup]"
-cp report.html ${LOG_DIR}/
-cp coverage.xml ${LOG_DIR}/
+ echo "##[group]Running ${test_file} in HPU compile mode..."
+ local ut_log_name="${LOG_DIR}/unittest_compile_${test_basename}.log"
+ PT_HPU_LAZY_MODE=0 pytest --mode compile --cov="${auto_round_path}" \
+ --cov-report term --html=report.html --self-contained-html \
+ --cov-report xml:coverage.xml --cov-append -vs --disable-warnings \
+ ${test_file} 2>&1 | tee ${ut_log_name}
+ echo "##[endgroup]"
+ done
+}
-if [ $(grep -c '== FAILURES ==' ${ut_log_name}) != 0 ] || [ $(grep -c '== ERRORS ==' ${ut_log_name}) != 0 ] || \
-[ $(grep -c 'Killed' ${ut_log_name}) != 0 ] || [ $(grep -c 'core dumped' ${ut_log_name}) != 0 ] || [ $(grep -c ' passed' ${ut_log_name}) == 0 ]; then
- echo "##[error]Find errors in pytest case, please check the output..."
- exit 1
-fi
+function print_summary() {
+ local status=0
+ while IFS= read -r line; do
+ if [[ "$line" == *"FAILED"* ]]; then
+ $LIGHT_RED && echo "$line" && $RESET
+ status=1
+ elif [[ "$line" == *"PASSED"* ]]; then
+ $LIGHT_GREEN && echo "$line" && $RESET
+ elif [[ "$line" == *"NO_TESTS"* ]]; then
+ $LIGHT_YELLOW && echo "$line" && $RESET
+ else
+ echo "$line"
+ fi
+ done < "${SUMMARY_LOG}"
+ exit $status
+}
-# if ut pass, collect the coverage file into artifacts
-cp .coverage ${LOG_DIR}/.coverage
+function collect_log() {
+ python /auto-round/.azure-pipelines/scripts/ut/collect_result.py \
+ --test-type "Unit Tests" --log-pattern "unittest_*.log" --log-dir ${LOG_DIR} --summary-log ${SUMMARY_LOG}
+ cp report.html ${LOG_DIR}/
+ cp coverage.xml ${LOG_DIR}/
+ cp .coverage "${LOG_DIR}/.coverage"
+}
-echo "UT finished successfully! "
\ No newline at end of file
+function main() {
+ setup_environment
+ run_unit_test
+ collect_log
+ print_summary
+}
+
+main
\ No newline at end of file
diff --git a/.azure-pipelines/scripts/ut/run_ut_xpu.sh b/.azure-pipelines/scripts/ut/run_ut_xpu.sh
index 117e70b09..9c2163f95 100644
--- a/.azure-pipelines/scripts/ut/run_ut_xpu.sh
+++ b/.azure-pipelines/scripts/ut/run_ut_xpu.sh
@@ -1,53 +1,93 @@
#!/bin/bash
-set -xe
-
-echo "##[group]set up UT env..."
-uv pip install pytest-cov pytest-html
-uv pip list
-echo "##[endgroup]"
-
-git config --global --add safe.directory /auto-round
-rm -rf /auto-round/auto_round
-cd /auto-round/test || exit 1
-
-echo "##[group]check xpu env..."
-echo "ZE_AFFINITY_MASK: ${ZE_AFFINITY_MASK}"
-python - <<'PY'
-import torch
-print("torch:", torch.__version__)
-print("xpu available:", torch.xpu.is_available())
-print("xpu count:", torch.xpu.device_count())
-PY
-echo "##[endgroup]"
-
-export TQDM_MININTERVAL=60
-export HF_HUB_DISABLE_PROGRESS_BARS=1
-export LD_LIBRARY_PATH=${HOME}/.venv/lib/:$LD_LIBRARY_PATH
-export COVERAGE_RCFILE=/auto-round/.azure-pipelines/scripts/ut/.coverage
-auto_round_path=$(python -c 'import auto_round; print(auto_round.__path__[0])')
-
-LOG_DIR=/auto-round/log_dir
-mkdir -p ${LOG_DIR}
-ut_log_name=${LOG_DIR}/ut.log
-
-find ./test_ark -name "test*.py" | sed "s,\.\/,python -m pytest --cov=\"${auto_round_path}\" --cov-report term --html=report.html --self-contained-html --cov-report xml:coverage.xml --cov-append -vs --disable-warnings ,g" > run_ark.sh
-cat run_ark.sh
-find ./test_xpu -name "test*.py" | sed "s,\.\/,python -m pytest --cov=\"${auto_round_path}\" --cov-report term --html=report.html --self-contained-html --cov-report xml:coverage.xml --cov-append -vs --disable-warnings ,g" > run_xpu.sh
-cat run_xpu.sh
-
-numactl --physcpubind="${NUMA_CPUSET:-0-27}" --membind="${NUMA_NODE:-0}" bash run_xpu.sh 2>&1 | tee "${ut_log_name}"
-
-numactl --physcpubind="${NUMA_CPUSET:-0-27}" --membind="${NUMA_NODE:-0}" bash run_ark.sh 2>&1 | tee -a "${ut_log_name}"
-
-cp report.html ${LOG_DIR}/
-cp coverage.xml ${LOG_DIR}/
-
-if [ $(grep -c '== FAILURES ==' ${ut_log_name}) != 0 ] || [ $(grep -c '== ERRORS ==' ${ut_log_name}) != 0 ] || \
-[ $(grep -c 'Killed' ${ut_log_name}) != 0 ] || [ $(grep -c 'core dumped' ${ut_log_name}) != 0 ] || [ $(grep -c ' passed' ${ut_log_name}) == 0 ]; then
- echo "##[error]Find errors in pytest case, please check the output..."
- exit 1
-fi
-
-cp .coverage "${LOG_DIR}/.coverage"
-
-echo "UT finished successfully! "
+set -e
+
+source /auto-round/.azure-pipelines/scripts/change_color.sh
+
+function setup_environment() {
+ echo "##[group]set up UT env..."
+ uv pip install pytest-cov pytest-html
+ uv pip list
+ echo "##[endgroup]"
+
+ git config --global --add safe.directory /auto-round
+ rm -rf /auto-round/auto_round
+ cd /auto-round/test || exit 1
+
+ echo "##[group]check xpu env..."
+ echo "ZE_AFFINITY_MASK: ${ZE_AFFINITY_MASK}"
+ python -c "import torch; print('torch:', torch.__version__); print('xpu available:', torch.xpu.is_available()); print('xpu count:', torch.xpu.device_count())"
+ echo "##[endgroup]"
+
+ export TZ='Asia/Shanghai'
+ export TQDM_MININTERVAL=60
+ export HF_HUB_DISABLE_PROGRESS_BARS=1
+ export LD_LIBRARY_PATH=${HOME}/.venv/lib/:$LD_LIBRARY_PATH
+ export COVERAGE_RCFILE=/auto-round/.azure-pipelines/scripts/ut/.coverage
+
+ LOG_DIR=/auto-round/log_dir
+ mkdir -p ${LOG_DIR}
+ ut_log_name=${LOG_DIR}/ut.log
+ SUMMARY_LOG="${LOG_DIR}/results_summary.log"
+}
+
+function run_unit_test() {
+ auto_round_path=$(python -c 'import auto_round; print(auto_round.__path__[0])')
+
+ for test_file in $(find ./test_ark -name "test*.py" | sort); do
+ local test_basename=$(basename ${test_file} .py)
+
+ echo "##[group]Running ark ${test_file}..."
+ local ut_log_name="${LOG_DIR}/unittest_ark_${test_basename}.log"
+ numactl --physcpubind="${NUMA_CPUSET:-0-27}" --membind="${NUMA_NODE:-0}" pytest --cov="${auto_round_path}" \
+ --cov-report term --html=report.html --self-contained-html \
+ --cov-report xml:coverage.xml --cov-append -vs --disable-warnings \
+ ${test_file} 2>&1 | tee ${ut_log_name}
+ echo "##[endgroup]"
+ done
+
+ for test_file in $(find ./test_xpu -name "test*.py" | sort); do
+ local test_basename=$(basename ${test_file} .py)
+
+ echo "##[group]Running xpu ${test_file}..."
+ local ut_log_name="${LOG_DIR}/unittest_xpu_${test_basename}.log"
+ numactl --physcpubind="${NUMA_CPUSET:-0-27}" --membind="${NUMA_NODE:-0}" pytest --cov="${auto_round_path}" \
+ --cov-report term --html=report.html --self-contained-html \
+ --cov-report xml:coverage.xml --cov-append -vs --disable-warnings \
+ ${test_file} 2>&1 | tee ${ut_log_name}
+ echo "##[endgroup]"
+ done
+}
+
+function print_summary() {
+ local status=0
+ while IFS= read -r line; do
+ if [[ "$line" == *"FAILED"* ]]; then
+ $LIGHT_RED && echo "$line" && $RESET
+ status=1
+ elif [[ "$line" == *"PASSED"* ]]; then
+ $LIGHT_GREEN && echo "$line" && $RESET
+ elif [[ "$line" == *"NO_TESTS"* ]]; then
+ $LIGHT_YELLOW && echo "$line" && $RESET
+ else
+ echo "$line"
+ fi
+ done < "${SUMMARY_LOG}"
+ exit $status
+}
+
+function collect_log() {
+ python /auto-round/.azure-pipelines/scripts/ut/collect_result.py \
+ --test-type "Unit Tests" --log-pattern "unittest_*.log" --log-dir ${LOG_DIR} --summary-log ${SUMMARY_LOG}
+ cp .coverage "${LOG_DIR}/.coverage"
+ cp report.html ${LOG_DIR}/
+ cp coverage.xml ${LOG_DIR}/
+}
+
+function main() {
+ setup_environment
+ run_unit_test
+ collect_log
+ print_summary
+}
+
+main