PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Refine CI (#1644)
Sun, Xuehao9f54985cd71f
diff --git a/.azure-pipelines/scripts/cuda_unit_test/runpod_manager.py b/.azure-pipelines/scripts/cuda_unit_test/runpod_manager.pyindex fcdfc8ab0..26faa1868 100644--- a/.azure-pipelines/scripts/cuda_unit_test/runpod_manager.py+++ b/.azure-pipelines/scripts/cuda_unit_test/runpod_manager.py@@ -7,7 +7,6 @@ import requestsTARGET_GPUS = ["NVIDIA GeForce RTX 4090",- "NVIDIA RTX PRO 4500 Blackwell","NVIDIA GeForce RTX 5090","NVIDIA RTX 6000 Ada Generation","NVIDIA L40S",@@ -47,90 +46,6 @@ REQUIRED_COUNT = 1IMAGES_NAME = "xuehaosu/azure-agent:v0.1"-def check_gpu_count(token):- url = "https://api.runpod.io/graphql"- ids_string = ", ".join([f'"{gid}"' for gid in TARGET_GPUS])- graphql_query = """- query GpuAvailability($input: GpuLowestPriceInput!) {- gpuTypes(input: {ids: [%s]}) {- id- displayName- memoryInGb- secureCloud- communityCloud- maxGpuCount- maxGpuCountSecureCloud- maxGpuCountCommunityCloud- minPodGpuCount- lowestPrice(input: $input) {- gpuName- stockStatus- minimumBidPrice- uninterruptablePrice- maxGpuCount- maxUnreservedGpuCount- availableGpuCounts- rentedCount- totalCount- }- }- }- """ % ids_string-- datacenter_id_string = ",".join(DATA_CENTER_SELECT_LIST)- variables = {- "input": {- "gpuCount": 1,- "secureCloud": True,- "minMemoryInGb": 0,- "minVcpuCount": 0,- "dataCenterId": datacenter_id_string,- }- }-- try:- response = requests.post(- url,- json={- "query": graphql_query,- "variables": variables,- },- headers={- "Content-Type": "application/json",- "Authorization": f"Bearer {token}",- },- )- response.raise_for_status()- data = response.json()- all_gpus = data.get("data", {}).get("gpuTypes", [])-- id_to_gpu = {gpu["id"]: gpu for gpu in all_gpus}- all_gpus = [id_to_gpu[gpu_id] for gpu_id in TARGET_GPUS if gpu_id in id_to_gpu]-- print("--- Checking target graphics card inventory ---\n")-- for gpu in all_gpus:- gpu_id = gpu.get("id")-- if gpu_id in TARGET_GPUS:- max_count = gpu.get("lowestPrice", {}).get("maxUnreservedGpuCount", 0)-- if REQUIRED_COUNT > max_count:- continue- else:- return gpu_id-- print("❌ No compliant GPU found.")- return None-- except requests.exceptions.RequestException as e:- print(f"HTTP Error: {e}")- sys.exit(1)- except Exception as e:- print(f"Error: {e}")- sys.exit(1)--def run_create_pod(api_key, payload):url = "https://rest.runpod.io/v1/pods"headers = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}@@ -141,17 +56,29 @@ def run_create_pod(api_key, payload):if response.status_code >= 500 or response.status_code == 400:if attempt < max_retries:- print(f"⚠️ {response.status_code} Error, Retrying in 60 seconds ({attempt + 1}/{max_retries})...")- time.sleep(60)+ print(f"⚠️ {response.status_code} Error, Retrying in 90 seconds ({attempt + 1}/{max_retries})...")+ time.sleep(90)continueelse:print(f"❌ {response.status_code} Error, Reached maximum retry attempts ({max_retries}), giving up.")- response.raise_for_status()- result = response.json()+ try:+ result = response.json()+ except ValueError:+ result = {}+if "errors" in result:- print("❌ Errors:")+ print("❌ API Errors:")print(json.dumps(result["errors"], indent=2))++ try:+ response.raise_for_status()+ except requests.exceptions.HTTPError as e:+ print(f"❌ HTTP Error: {e}")+ if result:+ print("Response payload:", json.dumps(result, indent=2))+ else:+ print("Response text:", response.text)sys.exit(1)return result@@ -161,19 +88,6 @@ def create_pod(args):if args.env:env_dict = {kv.split("=", 1)[0]: kv.split("=", 1)[1] for kv in args.env}- gpu_type = None- for _ in range(10): # Try up to 10 times to find an available GPU- gpu_type = check_gpu_count(args.api_key)- if gpu_type:- break- else:- print("⏳ No compliant GPU available. Retrying in 5 minutes...")- time.sleep(60 * 5)-- if not gpu_type:- print("❌ No compliant GPU found after multiple attempts. Exiting.")- sys.exit(1)-payload = {"cloudType": "SECURE","containerDiskInGb": args.container_disk_size,@@ -181,7 +95,8 @@ def create_pod(args):"dataCenterPriority": "availability","env": env_dict,"gpuCount": args.gpu_count,- "gpuTypeIds": [gpu_type],+ "gpuTypeIds": TARGET_GPUS,+ "gpuTypePriority": "custom","name": args.name,"volumeInGb": 0,"imageName": IMAGES_NAME,diff --git a/.azure-pipelines/scripts/ut/run_ut_hpu.sh b/.azure-pipelines/scripts/ut/run_ut_hpu.shindex 7a49c02e8..e8b36c1d3 100644--- a/.azure-pipelines/scripts/ut/run_ut_hpu.sh+++ b/.azure-pipelines/scripts/ut/run_ut_hpu.sh@@ -1,44 +1,84 @@#!/bin/bash-set -xe+set -e+source /auto-round/.azure-pipelines/scripts/change_color.sh-# install requirements-echo "set up UT env..."-export TQDM_MININTERVAL=60-export HF_HUB_DISABLE_PROGRESS_BARS=1-pip install pytest-cov pytest-html-pip list+function setup_environment() {+ # install requirements+ echo "##[group]set up UT env..."+ export TZ='Asia/Shanghai'+ export TQDM_MININTERVAL=60+ export HF_HUB_DISABLE_PROGRESS_BARS=1+ pip install pytest-cov pytest-html+ pip list+ echo "##[endgroup]"-rm -rf /auto-round/auto_round-cd /auto-round/test || exit 1+ rm -rf /auto-round/auto_round+ cd /auto-round/test || exit 1-export LD_LIBRARY_PATH=/usr/local/lib/:$LD_LIBRARY_PATH-export FORCE_BF16=1-export COVERAGE_RCFILE=/auto-round/.azure-pipelines/scripts/ut/.coverage-auto_round_path=$(python -c 'import auto_round; print(auto_round.__path__[0])')+ export LD_LIBRARY_PATH=/usr/local/lib/:$LD_LIBRARY_PATH+ export FORCE_BF16=1+ export COVERAGE_RCFILE=/auto-round/.azure-pipelines/scripts/ut/.coverage-LOG_DIR=/auto-round/log_dir-mkdir -p ${LOG_DIR}-ut_log_name=${LOG_DIR}/ut.log+ LOG_DIR=/auto-round/log_dir+ mkdir -p ${LOG_DIR}+ ut_log_name=${LOG_DIR}/ut.log+ SUMMARY_LOG="${LOG_DIR}/results_summary.log"+}-find ./test_hpu -name "test*.py" | sed "s,\.\/, PT_HPU_LAZY_MODE=1 python -m pytest --cov=\"${auto_round_path}\" --cov-report term --html=report.html --self-contained-html --cov-report xml:coverage.xml --cov-append -vs --disable-warnings ,g" > run_lazy.sh-find ./test_hpu -name "test*.py" | sed "s,\.\/, PT_HPU_LAZY_MODE=0 python -m pytest --mode compile --cov=\"${auto_round_path}\" --cov-report term --html=report.html --self-contained-html --cov-report xml:coverage.xml --cov-append -vs --disable-warnings ,g" > run_compile.sh+function run_unit_test() {+ auto_round_path=$(python -c 'import auto_round; print(auto_round.__path__[0])')-cat run_lazy.sh-bash run_lazy.sh 2>&1 | tee ${ut_log_name}+ for test_file in $(find ./test_hpu -name "test*.py" | sort); do+ local test_basename=$(basename ${test_file} .py)-cat run_compile.sh-bash run_compile.sh 2>&1 | tee -a ${ut_log_name}+ echo "##[group]Running ${test_file} in HPU lazy mode..."+ local ut_log_name="${LOG_DIR}/unittest_lazy_${test_basename}.log"+ PT_HPU_LAZY_MODE=1 pytest --cov="${auto_round_path}" \+ --cov-report term --html=report.html --self-contained-html \+ --cov-report xml:coverage.xml --cov-append -vs --disable-warnings \+ ${test_file} 2>&1 | tee ${ut_log_name}+ echo "##[endgroup]"-cp report.html ${LOG_DIR}/-cp coverage.xml ${LOG_DIR}/+ echo "##[group]Running ${test_file} in HPU compile mode..."+ local ut_log_name="${LOG_DIR}/unittest_compile_${test_basename}.log"+ PT_HPU_LAZY_MODE=0 pytest --mode compile --cov="${auto_round_path}" \+ --cov-report term --html=report.html --self-contained-html \+ --cov-report xml:coverage.xml --cov-append -vs --disable-warnings \+ ${test_file} 2>&1 | tee ${ut_log_name}+ echo "##[endgroup]"+ done+}-if [ $(grep -c '== FAILURES ==' ${ut_log_name}) != 0 ] || [ $(grep -c '== ERRORS ==' ${ut_log_name}) != 0 ] || \-[ $(grep -c 'Killed' ${ut_log_name}) != 0 ] || [ $(grep -c 'core dumped' ${ut_log_name}) != 0 ] || [ $(grep -c ' passed' ${ut_log_name}) == 0 ]; then- echo "##[error]Find errors in pytest case, please check the output..."- exit 1-fi+function print_summary() {+ local status=0+ while IFS= read -r line; do+ if [[ "$line" == *"FAILED"* ]]; then+ $LIGHT_RED && echo "$line" && $RESET+ status=1+ elif [[ "$line" == *"PASSED"* ]]; then+ $LIGHT_GREEN && echo "$line" && $RESET+ elif [[ "$line" == *"NO_TESTS"* ]]; then+ $LIGHT_YELLOW && echo "$line" && $RESET+ else+ echo "$line"+ fi+ done < "${SUMMARY_LOG}"+ exit $status+}-# if ut pass, collect the coverage file into artifacts-cp .coverage ${LOG_DIR}/.coverage+function collect_log() {+ python /auto-round/.azure-pipelines/scripts/ut/collect_result.py \+ --test-type "Unit Tests" --log-pattern "unittest_*.log" --log-dir ${LOG_DIR} --summary-log ${SUMMARY_LOG}+ cp report.html ${LOG_DIR}/+ cp coverage.xml ${LOG_DIR}/+ cp .coverage "${LOG_DIR}/.coverage"+}-echo "UT finished successfully! "\ No newline at end of file+function main() {+ setup_environment+ run_unit_test+ collect_log+ print_summary+}++main\ No newline at end of filediff --git a/.azure-pipelines/scripts/ut/run_ut_xpu.sh b/.azure-pipelines/scripts/ut/run_ut_xpu.shindex 117e70b09..9c2163f95 100644--- a/.azure-pipelines/scripts/ut/run_ut_xpu.sh+++ b/.azure-pipelines/scripts/ut/run_ut_xpu.sh@@ -1,53 +1,93 @@#!/bin/bash-set -xe--echo "##[group]set up UT env..."-uv pip install pytest-cov pytest-html-uv pip list-echo "##[endgroup]"--git config --global --add safe.directory /auto-round-rm -rf /auto-round/auto_round-cd /auto-round/test || exit 1--echo "##[group]check xpu env..."-echo "ZE_AFFINITY_MASK: ${ZE_AFFINITY_MASK}"-python - <<'PY'-import torch-print("torch:", torch.__version__)-print("xpu available:", torch.xpu.is_available())-print("xpu count:", torch.xpu.device_count())-PY-echo "##[endgroup]"--export TQDM_MININTERVAL=60-export HF_HUB_DISABLE_PROGRESS_BARS=1-export LD_LIBRARY_PATH=${HOME}/.venv/lib/:$LD_LIBRARY_PATH-export COVERAGE_RCFILE=/auto-round/.azure-pipelines/scripts/ut/.coverage-auto_round_path=$(python -c 'import auto_round; print(auto_round.__path__[0])')--LOG_DIR=/auto-round/log_dir-mkdir -p ${LOG_DIR}-ut_log_name=${LOG_DIR}/ut.log--find ./test_ark -name "test*.py" | sed "s,\.\/,python -m pytest --cov=\"${auto_round_path}\" --cov-report term --html=report.html --self-contained-html --cov-report xml:coverage.xml --cov-append -vs --disable-warnings ,g" > run_ark.sh-cat run_ark.sh-find ./test_xpu -name "test*.py" | sed "s,\.\/,python -m pytest --cov=\"${auto_round_path}\" --cov-report term --html=report.html --self-contained-html --cov-report xml:coverage.xml --cov-append -vs --disable-warnings ,g" > run_xpu.sh-cat run_xpu.sh--numactl --physcpubind="${NUMA_CPUSET:-0-27}" --membind="${NUMA_NODE:-0}" bash run_xpu.sh 2>&1 | tee "${ut_log_name}"--numactl --physcpubind="${NUMA_CPUSET:-0-27}" --membind="${NUMA_NODE:-0}" bash run_ark.sh 2>&1 | tee -a "${ut_log_name}"--cp report.html ${LOG_DIR}/-cp coverage.xml ${LOG_DIR}/--if [ $(grep -c '== FAILURES ==' ${ut_log_name}) != 0 ] || [ $(grep -c '== ERRORS ==' ${ut_log_name}) != 0 ] || \-[ $(grep -c 'Killed' ${ut_log_name}) != 0 ] || [ $(grep -c 'core dumped' ${ut_log_name}) != 0 ] || [ $(grep -c ' passed' ${ut_log_name}) == 0 ]; then- echo "##[error]Find errors in pytest case, please check the output..."- exit 1-fi--cp .coverage "${LOG_DIR}/.coverage"--echo "UT finished successfully! "+set -e++source /auto-round/.azure-pipelines/scripts/change_color.sh++function setup_environment() {+ echo "##[group]set up UT env..."+ uv pip install pytest-cov pytest-html+ uv pip list+ echo "##[endgroup]"++ git config --global --add safe.directory /auto-round+ rm -rf /auto-round/auto_round+ cd /auto-round/test || exit 1++ echo "##[group]check xpu env..."+ echo "ZE_AFFINITY_MASK: ${ZE_AFFINITY_MASK}"+ python -c "import torch; print('torch:', torch.__version__); print('xpu available:', torch.xpu.is_available()); print('xpu count:', torch.xpu.device_count())"+ echo "##[endgroup]"++ export TZ='Asia/Shanghai'+ export TQDM_MININTERVAL=60+ export HF_HUB_DISABLE_PROGRESS_BARS=1+ export LD_LIBRARY_PATH=${HOME}/.venv/lib/:$LD_LIBRARY_PATH+ export COVERAGE_RCFILE=/auto-round/.azure-pipelines/scripts/ut/.coverage++ LOG_DIR=/auto-round/log_dir+ mkdir -p ${LOG_DIR}+ ut_log_name=${LOG_DIR}/ut.log+ SUMMARY_LOG="${LOG_DIR}/results_summary.log"+}++function run_unit_test() {+ auto_round_path=$(python -c 'import auto_round; print(auto_round.__path__[0])')++ for test_file in $(find ./test_ark -name "test*.py" | sort); do+ local test_basename=$(basename ${test_file} .py)++ echo "##[group]Running ark ${test_file}..."+ local ut_log_name="${LOG_DIR}/unittest_ark_${test_basename}.log"+ numactl --physcpubind="${NUMA_CPUSET:-0-27}" --membind="${NUMA_NODE:-0}" pytest --cov="${auto_round_path}" \+ --cov-report term --html=report.html --self-contained-html \+ --cov-report xml:coverage.xml --cov-append -vs --disable-warnings \+ ${test_file} 2>&1 | tee ${ut_log_name}+ echo "##[endgroup]"+ done++ for test_file in $(find ./test_xpu -name "test*.py" | sort); do+ local test_basename=$(basename ${test_file} .py)++ echo "##[group]Running xpu ${test_file}..."+ local ut_log_name="${LOG_DIR}/unittest_xpu_${test_basename}.log"+ numactl --physcpubind="${NUMA_CPUSET:-0-27}" --membind="${NUMA_NODE:-0}" pytest --cov="${auto_round_path}" \+ --cov-report term --html=report.html --self-contained-html \+ --cov-report xml:coverage.xml --cov-append -vs --disable-warnings \+ ${test_file} 2>&1 | tee ${ut_log_name}+ echo "##[endgroup]"+ done+}++function print_summary() {+ local status=0+ while IFS= read -r line; do+ if [[ "$line" == *"FAILED"* ]]; then+ $LIGHT_RED && echo "$line" && $RESET+ status=1+ elif [[ "$line" == *"PASSED"* ]]; then+ $LIGHT_GREEN && echo "$line" && $RESET+ elif [[ "$line" == *"NO_TESTS"* ]]; then+ $LIGHT_YELLOW && echo "$line" && $RESET+ else+ echo "$line"+ fi+ done < "${SUMMARY_LOG}"+ exit $status+}++function collect_log() {+ python /auto-round/.azure-pipelines/scripts/ut/collect_result.py \+ --test-type "Unit Tests" --log-pattern "unittest_*.log" --log-dir ${LOG_DIR} --summary-log ${SUMMARY_LOG}+ cp .coverage "${LOG_DIR}/.coverage"+ cp report.html ${LOG_DIR}/+ cp coverage.xml ${LOG_DIR}/+}++function main() {+ setup_environment+ run_unit_test+ collect_log+ print_summary+}++main