PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Enhance performance test (#1610)
Sun, Xuehao5b0d3faaf43b
diff --git a/.azure-pipelines/performance-test.yaml b/.azure-pipelines/performance-test.yamlindex b38a4e0af..b9676ff2d 100644--- a/.azure-pipelines/performance-test.yaml+++ b/.azure-pipelines/performance-test.yaml@@ -19,8 +19,6 @@ pr:variables:CONTAINER_NAME: "AutoRoundPerfTest"-pool: GAUDI-stages:- stage: PerformanceTestdisplayName: Performance Test@@ -28,6 +26,17 @@ stages:jobs:- job: PerformanceTestJobdisplayName: Performance Test Job+ pool: GAUDI++ strategy:+ matrix:+ Qwen3_W4A16:+ model_name: 'Qwen/Qwen3-0.6B'+ scheme: 'W4A16'+ Qwen3_FP8_STATIC:+ model_name: 'Qwen/Qwen3-0.6B'+ scheme: 'FP8_STATIC'+steps:- template: template/docker-template.ymlparameters:@@ -47,16 +56,9 @@ stages:- script: |docker exec $(CONTAINER_NAME) \bash -c "cd /auto-round/.azure-pipelines/scripts/performance \- && bash perf_test.sh"+ && bash perf_test.sh --model_name=$(model_name) --scheme=$(scheme)"displayName: "Run performance test"- - task: PublishPipelineArtifact@1- condition: succeededOrFailed()- inputs:- targetPath: $(Build.SourcesDirectory)/log_dir- artifact: "Performance_Test_Logs"- publishLocation: "pipeline"-- task: Bash@3condition: always()inputs:diff --git a/.azure-pipelines/scripts/performance/check_performance.py b/.azure-pipelines/scripts/performance/check_performance.pyindex 079b3b0a7..bdb74f6e8 100644--- a/.azure-pipelines/scripts/performance/check_performance.py+++ b/.azure-pipelines/scripts/performance/check_performance.py@@ -1,65 +1,127 @@+import loggingimport reimport sys+from dataclasses import dataclass+from pathlib import Path+from typing import Dict, Optional-LOG_DIR = "/auto-round/log_dir"+logging.basicConfig(level=logging.INFO, format="%(message)s")+LOG_DIR = Path("/auto-round/log_dir")+OUTPUT_BASE_DIR = Path("/auto-round/.azure-pipelines/scripts/performance")-def parse_tuning_time(log_file):- with open(log_file, "r") as f:- content = f.read()- pattern = r"tuning time ([0-9]+\.[0-9]+)"- match = re.search(pattern, content)+@dataclass+class QuantMetrics:+ tuning_time_s: Optional[float] = None+ peak_ram_gb: Optional[float] = None+ peak_vram_gb: Optional[float] = None+ output_size_gb: Optional[float] = None- if match:- elapsed = str_to_float(match.group(1))- return elapsed- return None+def get_dir_size_gb(path: Path) -> float:+ if not path.exists() or not path.is_dir():+ return 0.0-def str_to_float(value):- try:- return round(float(value), 4)- except ValueError:- return value+ total_bytes = sum(f.stat().st_size for f in path.rglob("*") if f.is_file())+ return round(total_bytes / (1024**3), 4)-def get_tuning_time():+def parse_log_file(log_file: Path) -> QuantMetrics:+ metrics = QuantMetrics()++ if not log_file.exists():+ logging.warning(f"Log file not found: {log_file}")+ return metrics++ content = log_file.read_text(encoding="utf-8")++ # Use findall to capture all occurrences and take the most recent one.+ time_matches = re.findall(r"tuning time ([0-9]+\.[0-9]+)", content)+ if time_matches:+ metrics.tuning_time_s = round(float(time_matches[-1]), 4)++ ram_matches = re.findall(r"'peak_ram':\s*([\d.]+)\s*GB,\s*'peak_vram':\s*([\d.]+)\s*GB", content)+ if ram_matches:+ last_ram, last_vram = ram_matches[-1]+ metrics.peak_ram_gb = round(float(last_ram), 4)+ metrics.peak_vram_gb = round(float(last_vram), 4)++ return metrics+++def get_tuning_info() -> Dict[str, Dict[str, QuantMetrics]]:summary = {}model_list = ["Qwen/Qwen3-0.6B"]+for model in model_list:summary[model] = {}for test_mode in ["current", "baseline"]:- log_file = f"{LOG_DIR}/perf_test_{test_mode}.log"- print(f"Processing {log_file}...")- tuning_time = parse_tuning_time(log_file)- if tuning_time is not None:- summary[model][test_mode] = tuning_time- else:- summary[model][test_mode] = "N/A"+ log_file = LOG_DIR / f"perf_test_{test_mode}.log"+ output_dir = OUTPUT_BASE_DIR / test_mode++ logging.info(f"Processing {log_file}...")++ metrics = parse_log_file(log_file)+ metrics.output_size_gb = get_dir_size_gb(output_dir)++ summary[model][test_mode] = metricsreturn summary+def compare_metric(+ metric_name: str, current: Optional[float], baseline: Optional[float], tolerance: float = 0.1+) -> bool:+ if current is None or baseline is None:+ logging.error(f" [-] {metric_name}: Incomplete data (Current: {current}, Baseline: {baseline})")+ return False++ if baseline == 0:+ logging.warning(f" [!] {metric_name}: Baseline is 0, cannot calculate ratio.")+ return False++ ratio = current / baseline+ diff_percent = (ratio - 1) * 100++ msg = f" [*] {metric_name:<20}: Current = {current:<8} | Baseline = {baseline:<8} (Diff: {diff_percent:+.2f}%)"++ if 1.0 - tolerance <= ratio <= 1.0 + tolerance:+ logging.info(f"{msg} -> PASS")+ return True+ else:+ logging.error(f"{msg} -> FAIL")+ return False++def check_performance():- status = True- summary = get_tuning_time()- for model, times in summary.items():- current_time = times.get("current", "N/A")- baseline_time = times.get("baseline", "N/A")- if current_time != "N/A" and baseline_time != "N/A":- print(f"{model}:\n Current = {current_time} seconds\n Baseline = {baseline_time} seconds")- ratio = current_time / baseline_time- if ratio < 0.9 or ratio > 1.1:- status = False- else:- print(f"{model}: Tuning time data is incomplete.")- status = False-- if status:- print("Performance check passed: Current tuning times are within acceptable limits compared to baseline.")+ summary = get_tuning_info()+ all_passed = True++ for model, modes in summary.items():+ logging.info(f"\nEvaluating Model: {model}")+ logging.info("-" * 60)++ current: QuantMetrics = modes.get("current", QuantMetrics())+ baseline: QuantMetrics = modes.get("baseline", QuantMetrics())++ if not compare_metric("Tuning Time (s)", current.tuning_time_s, baseline.tuning_time_s, tolerance=0.05):+ all_passed = False++ if not compare_metric("Peak RAM (GB)", current.peak_ram_gb, baseline.peak_ram_gb, tolerance=0.03):+ all_passed = False++ if not compare_metric("Peak VRAM (GB)", current.peak_vram_gb, baseline.peak_vram_gb, tolerance=0.03):+ all_passed = False++ if not compare_metric("Output Size (GB)", current.output_size_gb, baseline.output_size_gb, tolerance=0.01):+ all_passed = False++ logging.info("=" * 60)+ if all_passed:+ logging.info("✅ Performance check passed: All metrics are within acceptable limits.")else:- print("Performance check failed: Current tuning times exceed acceptable limits compared to baseline.")+ logging.error("❌ Performance check failed: Current metrics exceed acceptable limits compared to baseline.")sys.exit(1)diff --git a/.azure-pipelines/scripts/performance/perf_test.sh b/.azure-pipelines/scripts/performance/perf_test.shindex f24c1aac6..12e78906b 100644--- a/.azure-pipelines/scripts/performance/perf_test.sh+++ b/.azure-pipelines/scripts/performance/perf_test.sh@@ -1,59 +1,115 @@#!/bin/bash-set -e+set -euo pipefail++PATTERN='[-a-zA-Z0-9_]*='++for i in "$@"; do+ case $i in+ --model_name=*)+ model_name=$(echo $i | sed "s/${PATTERN}//")+ ;;+ --scheme=*)+ scheme=$(echo $i | sed "s/${PATTERN}//")+ ;;+ *)+ echo "Parameter $i not recognized."+ exit 1+ ;;+ esac+done++readonly WORKSPACE_DIR="/auto-round"+readonly LOG_DIR="${WORKSPACE_DIR}/log_dir"+readonly PERF_SCRIPT_DIR="${WORKSPACE_DIR}/.azure-pipelines/scripts/performance"+readonly BASELINE_GIT_URL="git+https://github.com/intel/auto-round.git"+readonly ITERS=200++log_group_start() { echo "##[group]$1"; }+log_group_end() { echo "##[endgroup]"; }+log_info() { echo -e "[\033[32mINFO\033[0m] $1"; }+log_err() { echo -e "[\033[31mERROR\033[0m] $1" >&2; }function setup_environment() {- echo "##[group]set up environment..."+ log_group_start "Set up environment..."+export TZ='Asia/Shanghai'export TQDM_MININTERVAL=60export HF_HUB_DISABLE_PROGRESS_BARS=1export UV_NO_PROGRESS=1export UV_SYSTEM_PYTHON=1- LOG_DIR="/auto-round/log_dir"+ log_info "Creating log directory: ${LOG_DIR}"mkdir -p "${LOG_DIR}"- model_name="Qwen/Qwen3-0.6B"- hf download ${model_name}- echo "##[endgroup]"-}+ log_info "Downloading model: ${model_name}"+ hf download "${model_name}"-function install_requirements() {- echo "##[group]install requirements..."- cd /auto-round- uv pip uninstall auto-round || true- BUILD_HPU_ONLY=1 uv pip install .- echo "##[endgroup]"+ log_group_end}-function install_baseline_requirements() {- echo "##[group]install baseline requirements..."- cd /auto-round- uv pip uninstall auto-round || true- BUILD_HPU_ONLY=1 uv pip install git+https://github.com/intel/auto-round.git- echo "##[endgroup]"+function install_auto_round() {+ local install_source=$1+ local mode_name=$2++ log_group_start "Install requirements for [${mode_name}]..."++ (+ cd "${WORKSPACE_DIR}"+ log_info "Uninstalling existing auto-round..."+ uv pip uninstall auto-round || true++ log_info "Installing auto-round from: ${install_source}"+ BUILD_HPU_ONLY=1 uv pip install "${install_source}"+ )++ log_group_end}function run_performance_test() {- test_mode=$1- cd /auto-round/.azure-pipelines/scripts/performance- local log_file="perf_test_${test_mode}.log"- rm -rf "saved" "${LOG_DIR}/${log_file}"- echo "##[group]run ${test_mode} performance test..."- auto-round --model_name ${model_name} --bits 4 --iters 200 --enable_torch_compile --device hpu --output_dir ./saved 2>&1 | tee -a "${LOG_DIR}/${log_file}"- echo "##[endgroup]"+ local test_mode=$1+ local log_file="${LOG_DIR}/perf_test_${test_mode}.log"++ log_group_start "Run ${test_mode} performance test (${scheme})..."++ (+ cd "${PERF_SCRIPT_DIR}"+ log_info "Executing auto-round for ${scheme}. Logging to ${log_file}"+ auto-round \+ --model_name "${model_name}" \+ --scheme "${scheme}" \+ --iters "${ITERS}" \+ --enable_torch_compile \+ --device hpu \+ --output_dir "./${test_mode}" 2>&1 | tee -a "${log_file}"+ )++ log_group_end+}++function run_performance_check() {+ log_group_start "Check performance results..."++ (+ cd "${PERF_SCRIPT_DIR}"+ log_info "Executing check_performance.py"+ python check_performance.py+ )++ log_group_end}function main() {setup_environment- install_requirements+ install_auto_round "." "current"run_performance_test "current"- install_baseline_requirements+ install_auto_round "${BASELINE_GIT_URL}" "baseline"run_performance_test "baseline"- cd /auto-round/.azure-pipelines/scripts/performance- python check_performance.py+ run_performance_check++ log_info "All tasks completed successfully."}-main+main "$@"\ No newline at end of file