Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Enhance performance test (#1610)

Sun, Xuehao5b0d3faaf43b
diff --git a/.azure-pipelines/performance-test.yaml b/.azure-pipelines/performance-test.yaml
index b38a4e0af..b9676ff2d 100644
--- a/.azure-pipelines/performance-test.yaml
+++ b/.azure-pipelines/performance-test.yaml
@@ -19,8 +19,6 @@ pr:
variables:
CONTAINER_NAME: "AutoRoundPerfTest"
-pool: GAUDI
-
stages:
- stage: PerformanceTest
displayName: Performance Test
@@ -28,6 +26,17 @@ stages:
jobs:
- job: PerformanceTestJob
displayName: Performance Test Job
+ pool: GAUDI
+
+ strategy:
+ matrix:
+ Qwen3_W4A16:
+ model_name: 'Qwen/Qwen3-0.6B'
+ scheme: 'W4A16'
+ Qwen3_FP8_STATIC:
+ model_name: 'Qwen/Qwen3-0.6B'
+ scheme: 'FP8_STATIC'
+
steps:
- template: template/docker-template.yml
parameters:
@@ -47,16 +56,9 @@ stages:
- script: |
docker exec $(CONTAINER_NAME) \
bash -c "cd /auto-round/.azure-pipelines/scripts/performance \
- && bash perf_test.sh"
+ && bash perf_test.sh --model_name=$(model_name) --scheme=$(scheme)"
displayName: "Run performance test"
- - task: PublishPipelineArtifact@1
- condition: succeededOrFailed()
- inputs:
- targetPath: $(Build.SourcesDirectory)/log_dir
- artifact: "Performance_Test_Logs"
- publishLocation: "pipeline"
-
- task: Bash@3
condition: always()
inputs:
diff --git a/.azure-pipelines/scripts/performance/check_performance.py b/.azure-pipelines/scripts/performance/check_performance.py
index 079b3b0a7..bdb74f6e8 100644
--- a/.azure-pipelines/scripts/performance/check_performance.py
+++ b/.azure-pipelines/scripts/performance/check_performance.py
@@ -1,65 +1,127 @@
+import logging
import re
import sys
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, Optional
-LOG_DIR = "/auto-round/log_dir"
+logging.basicConfig(level=logging.INFO, format="%(message)s")
+LOG_DIR = Path("/auto-round/log_dir")
+OUTPUT_BASE_DIR = Path("/auto-round/.azure-pipelines/scripts/performance")
-def parse_tuning_time(log_file):
- with open(log_file, "r") as f:
- content = f.read()
- pattern = r"tuning time ([0-9]+\.[0-9]+)"
- match = re.search(pattern, content)
+@dataclass
+class QuantMetrics:
+ tuning_time_s: Optional[float] = None
+ peak_ram_gb: Optional[float] = None
+ peak_vram_gb: Optional[float] = None
+ output_size_gb: Optional[float] = None
- if match:
- elapsed = str_to_float(match.group(1))
- return elapsed
- return None
+def get_dir_size_gb(path: Path) -> float:
+ if not path.exists() or not path.is_dir():
+ return 0.0
-def str_to_float(value):
- try:
- return round(float(value), 4)
- except ValueError:
- return value
+ total_bytes = sum(f.stat().st_size for f in path.rglob("*") if f.is_file())
+ return round(total_bytes / (1024**3), 4)
-def get_tuning_time():
+def parse_log_file(log_file: Path) -> QuantMetrics:
+ metrics = QuantMetrics()
+
+ if not log_file.exists():
+ logging.warning(f"Log file not found: {log_file}")
+ return metrics
+
+ content = log_file.read_text(encoding="utf-8")
+
+ # Use findall to capture all occurrences and take the most recent one.
+ time_matches = re.findall(r"tuning time ([0-9]+\.[0-9]+)", content)
+ if time_matches:
+ metrics.tuning_time_s = round(float(time_matches[-1]), 4)
+
+ ram_matches = re.findall(r"'peak_ram':\s*([\d.]+)\s*GB,\s*'peak_vram':\s*([\d.]+)\s*GB", content)
+ if ram_matches:
+ last_ram, last_vram = ram_matches[-1]
+ metrics.peak_ram_gb = round(float(last_ram), 4)
+ metrics.peak_vram_gb = round(float(last_vram), 4)
+
+ return metrics
+
+
+def get_tuning_info() -> Dict[str, Dict[str, QuantMetrics]]:
summary = {}
model_list = ["Qwen/Qwen3-0.6B"]
+
for model in model_list:
summary[model] = {}
for test_mode in ["current", "baseline"]:
- log_file = f"{LOG_DIR}/perf_test_{test_mode}.log"
- print(f"Processing {log_file}...")
- tuning_time = parse_tuning_time(log_file)
- if tuning_time is not None:
- summary[model][test_mode] = tuning_time
- else:
- summary[model][test_mode] = "N/A"
+ log_file = LOG_DIR / f"perf_test_{test_mode}.log"
+ output_dir = OUTPUT_BASE_DIR / test_mode
+
+ logging.info(f"Processing {log_file}...")
+
+ metrics = parse_log_file(log_file)
+ metrics.output_size_gb = get_dir_size_gb(output_dir)
+
+ summary[model][test_mode] = metrics
return summary
+def compare_metric(
+ metric_name: str, current: Optional[float], baseline: Optional[float], tolerance: float = 0.1
+) -> bool:
+ if current is None or baseline is None:
+ logging.error(f" [-] {metric_name}: Incomplete data (Current: {current}, Baseline: {baseline})")
+ return False
+
+ if baseline == 0:
+ logging.warning(f" [!] {metric_name}: Baseline is 0, cannot calculate ratio.")
+ return False
+
+ ratio = current / baseline
+ diff_percent = (ratio - 1) * 100
+
+ msg = f" [*] {metric_name:<20}: Current = {current:<8} | Baseline = {baseline:<8} (Diff: {diff_percent:+.2f}%)"
+
+ if 1.0 - tolerance <= ratio <= 1.0 + tolerance:
+ logging.info(f"{msg} -> PASS")
+ return True
+ else:
+ logging.error(f"{msg} -> FAIL")
+ return False
+
+
def check_performance():
- status = True
- summary = get_tuning_time()
- for model, times in summary.items():
- current_time = times.get("current", "N/A")
- baseline_time = times.get("baseline", "N/A")
- if current_time != "N/A" and baseline_time != "N/A":
- print(f"{model}:\n Current = {current_time} seconds\n Baseline = {baseline_time} seconds")
- ratio = current_time / baseline_time
- if ratio < 0.9 or ratio > 1.1:
- status = False
- else:
- print(f"{model}: Tuning time data is incomplete.")
- status = False
-
- if status:
- print("Performance check passed: Current tuning times are within acceptable limits compared to baseline.")
+ summary = get_tuning_info()
+ all_passed = True
+
+ for model, modes in summary.items():
+ logging.info(f"\nEvaluating Model: {model}")
+ logging.info("-" * 60)
+
+ current: QuantMetrics = modes.get("current", QuantMetrics())
+ baseline: QuantMetrics = modes.get("baseline", QuantMetrics())
+
+ if not compare_metric("Tuning Time (s)", current.tuning_time_s, baseline.tuning_time_s, tolerance=0.05):
+ all_passed = False
+
+ if not compare_metric("Peak RAM (GB)", current.peak_ram_gb, baseline.peak_ram_gb, tolerance=0.03):
+ all_passed = False
+
+ if not compare_metric("Peak VRAM (GB)", current.peak_vram_gb, baseline.peak_vram_gb, tolerance=0.03):
+ all_passed = False
+
+ if not compare_metric("Output Size (GB)", current.output_size_gb, baseline.output_size_gb, tolerance=0.01):
+ all_passed = False
+
+ logging.info("=" * 60)
+ if all_passed:
+ logging.info("✅ Performance check passed: All metrics are within acceptable limits.")
else:
- print("Performance check failed: Current tuning times exceed acceptable limits compared to baseline.")
+ logging.error("❌ Performance check failed: Current metrics exceed acceptable limits compared to baseline.")
sys.exit(1)
diff --git a/.azure-pipelines/scripts/performance/perf_test.sh b/.azure-pipelines/scripts/performance/perf_test.sh
index f24c1aac6..12e78906b 100644
--- a/.azure-pipelines/scripts/performance/perf_test.sh
+++ b/.azure-pipelines/scripts/performance/perf_test.sh
@@ -1,59 +1,115 @@
#!/bin/bash
-set -e
+set -euo pipefail
+
+PATTERN='[-a-zA-Z0-9_]*='
+
+for i in "$@"; do
+ case $i in
+ --model_name=*)
+ model_name=$(echo $i | sed "s/${PATTERN}//")
+ ;;
+ --scheme=*)
+ scheme=$(echo $i | sed "s/${PATTERN}//")
+ ;;
+ *)
+ echo "Parameter $i not recognized."
+ exit 1
+ ;;
+ esac
+done
+
+readonly WORKSPACE_DIR="/auto-round"
+readonly LOG_DIR="${WORKSPACE_DIR}/log_dir"
+readonly PERF_SCRIPT_DIR="${WORKSPACE_DIR}/.azure-pipelines/scripts/performance"
+readonly BASELINE_GIT_URL="git+https://github.com/intel/auto-round.git"
+readonly ITERS=200
+
+log_group_start() { echo "##[group]$1"; }
+log_group_end() { echo "##[endgroup]"; }
+log_info() { echo -e "[\033[32mINFO\033[0m] $1"; }
+log_err() { echo -e "[\033[31mERROR\033[0m] $1" >&2; }
function setup_environment() {
- echo "##[group]set up environment..."
+ log_group_start "Set up environment..."
+
export TZ='Asia/Shanghai'
export TQDM_MININTERVAL=60
export HF_HUB_DISABLE_PROGRESS_BARS=1
export UV_NO_PROGRESS=1
export UV_SYSTEM_PYTHON=1
- LOG_DIR="/auto-round/log_dir"
+ log_info "Creating log directory: ${LOG_DIR}"
mkdir -p "${LOG_DIR}"
- model_name="Qwen/Qwen3-0.6B"
- hf download ${model_name}
- echo "##[endgroup]"
-}
+ log_info "Downloading model: ${model_name}"
+ hf download "${model_name}"
-function install_requirements() {
- echo "##[group]install requirements..."
- cd /auto-round
- uv pip uninstall auto-round || true
- BUILD_HPU_ONLY=1 uv pip install .
- echo "##[endgroup]"
+ log_group_end
}
-function install_baseline_requirements() {
- echo "##[group]install baseline requirements..."
- cd /auto-round
- uv pip uninstall auto-round || true
- BUILD_HPU_ONLY=1 uv pip install git+https://github.com/intel/auto-round.git
- echo "##[endgroup]"
+function install_auto_round() {
+ local install_source=$1
+ local mode_name=$2
+
+ log_group_start "Install requirements for [${mode_name}]..."
+
+ (
+ cd "${WORKSPACE_DIR}"
+ log_info "Uninstalling existing auto-round..."
+ uv pip uninstall auto-round || true
+
+ log_info "Installing auto-round from: ${install_source}"
+ BUILD_HPU_ONLY=1 uv pip install "${install_source}"
+ )
+
+ log_group_end
}
function run_performance_test() {
- test_mode=$1
- cd /auto-round/.azure-pipelines/scripts/performance
- local log_file="perf_test_${test_mode}.log"
- rm -rf "saved" "${LOG_DIR}/${log_file}"
- echo "##[group]run ${test_mode} performance test..."
- auto-round --model_name ${model_name} --bits 4 --iters 200 --enable_torch_compile --device hpu --output_dir ./saved 2>&1 | tee -a "${LOG_DIR}/${log_file}"
- echo "##[endgroup]"
+ local test_mode=$1
+ local log_file="${LOG_DIR}/perf_test_${test_mode}.log"
+
+ log_group_start "Run ${test_mode} performance test (${scheme})..."
+
+ (
+ cd "${PERF_SCRIPT_DIR}"
+ log_info "Executing auto-round for ${scheme}. Logging to ${log_file}"
+ auto-round \
+ --model_name "${model_name}" \
+ --scheme "${scheme}" \
+ --iters "${ITERS}" \
+ --enable_torch_compile \
+ --device hpu \
+ --output_dir "./${test_mode}" 2>&1 | tee -a "${log_file}"
+ )
+
+ log_group_end
+}
+
+function run_performance_check() {
+ log_group_start "Check performance results..."
+
+ (
+ cd "${PERF_SCRIPT_DIR}"
+ log_info "Executing check_performance.py"
+ python check_performance.py
+ )
+
+ log_group_end
}
function main() {
setup_environment
- install_requirements
+ install_auto_round "." "current"
run_performance_test "current"
- install_baseline_requirements
+ install_auto_round "${BASELINE_GIT_URL}" "baseline"
run_performance_test "baseline"
- cd /auto-round/.azure-pipelines/scripts/performance
- python check_performance.py
+ run_performance_check
+
+ log_info "All tasks completed successfully."
}
-main
+main "$@"
\ No newline at end of file