PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Add Performance Test (#1561)
Sun, Xuehao29f0275126cd
diff --git a/.azure-pipelines/performance-test.yaml b/.azure-pipelines/performance-test.yamlnew file mode 100644index 000000000..b38a4e0af--- /dev/null+++ b/.azure-pipelines/performance-test.yaml@@ -0,0 +1,67 @@+trigger: none++pr:+ autoCancel: true+ drafts: false+ branches:+ include:+ - main+ paths:+ include:+ - auto_round+ - auto_round_extension+ - .azure-pipelines/scripts/performance+ - .azure-pipelines/performance-test.yaml+ exclude:+ - "*.md"+ - "**/*.md"++variables:+ CONTAINER_NAME: "AutoRoundPerfTest"++pool: GAUDI++stages:+ - stage: PerformanceTest+ displayName: Performance Test+ dependsOn: []+ jobs:+ - job: PerformanceTestJob+ displayName: Performance Test Job+ steps:+ - template: template/docker-template.yml+ parameters:+ containerName: $(CONTAINER_NAME)+ imageSource: pull++ - script: |+ docker exec $(CONTAINER_NAME) bash -c "cd /auto-round \+ && export UV_NO_PROGRESS=1 \+ && export UV_SYSTEM_PYTHON=1 \+ && pip install uv \+ && uv pip install -r test/test_hpu/requirements.txt \+ && uv pip install huggingface_hub \+ && uv pip list"+ displayName: "HPU Env Setup"++ - script: |+ docker exec $(CONTAINER_NAME) \+ bash -c "cd /auto-round/.azure-pipelines/scripts/performance \+ && bash perf_test.sh"+ displayName: "Run performance test"++ - task: PublishPipelineArtifact@1+ condition: succeededOrFailed()+ inputs:+ targetPath: $(Build.SourcesDirectory)/log_dir+ artifact: "Performance_Test_Logs"+ publishLocation: "pipeline"++ - task: Bash@3+ condition: always()+ inputs:+ targetType: "inline"+ script: |+ docker stop $(CONTAINER_NAME)+ docker rm -vf $(CONTAINER_NAME) || true+ displayName: "Docker clean up"diff --git a/.azure-pipelines/scripts/performance/check_performance.py b/.azure-pipelines/scripts/performance/check_performance.pynew file mode 100644index 000000000..079b3b0a7--- /dev/null+++ b/.azure-pipelines/scripts/performance/check_performance.py@@ -0,0 +1,71 @@+import re+import sys++LOG_DIR = "/auto-round/log_dir"+++def parse_tuning_time(log_file):+ with open(log_file, "r") as f:+ content = f.read()++ pattern = r"tuning time ([0-9]+\.[0-9]+)"+ match = re.search(pattern, content)++ if match:+ elapsed = str_to_float(match.group(1))+ return elapsed+ return None+++def str_to_float(value):+ try:+ return round(float(value), 4)+ except ValueError:+ return value+++def get_tuning_time():+ summary = {}+ model_list = ["Qwen/Qwen3-0.6B"]+ for model in model_list:+ summary[model] = {}+ for test_mode in ["current", "baseline"]:+ log_file = f"{LOG_DIR}/perf_test_{test_mode}.log"+ print(f"Processing {log_file}...")+ tuning_time = parse_tuning_time(log_file)+ if tuning_time is not None:+ summary[model][test_mode] = tuning_time+ else:+ summary[model][test_mode] = "N/A"++ return summary+++def check_performance():+ status = True+ summary = get_tuning_time()+ for model, times in summary.items():+ current_time = times.get("current", "N/A")+ baseline_time = times.get("baseline", "N/A")+ if current_time != "N/A" and baseline_time != "N/A":+ print(f"{model}:\n Current = {current_time} seconds\n Baseline = {baseline_time} seconds")+ ratio = current_time / baseline_time+ if ratio < 0.9 or ratio > 1.1:+ status = False+ else:+ print(f"{model}: Tuning time data is incomplete.")+ status = False++ if status:+ print("Performance check passed: Current tuning times are within acceptable limits compared to baseline.")+ else:+ print("Performance check failed: Current tuning times exceed acceptable limits compared to baseline.")+ sys.exit(1)+++def main():+ check_performance()+++if __name__ == "__main__":+ main()diff --git a/.azure-pipelines/scripts/performance/perf_test.sh b/.azure-pipelines/scripts/performance/perf_test.shnew file mode 100644index 000000000..f24c1aac6--- /dev/null+++ b/.azure-pipelines/scripts/performance/perf_test.sh@@ -0,0 +1,59 @@+#!/bin/bash+set -e++function setup_environment() {+ echo "##[group]set up environment..."+ export TZ='Asia/Shanghai'+ export TQDM_MININTERVAL=60+ export HF_HUB_DISABLE_PROGRESS_BARS=1+ export UV_NO_PROGRESS=1+ export UV_SYSTEM_PYTHON=1++ LOG_DIR="/auto-round/log_dir"+ mkdir -p "${LOG_DIR}"++ model_name="Qwen/Qwen3-0.6B"+ hf download ${model_name}+ echo "##[endgroup]"+}++function install_requirements() {+ echo "##[group]install requirements..."+ cd /auto-round+ uv pip uninstall auto-round || true+ BUILD_HPU_ONLY=1 uv pip install .+ echo "##[endgroup]"+}++function install_baseline_requirements() {+ echo "##[group]install baseline requirements..."+ cd /auto-round+ uv pip uninstall auto-round || true+ BUILD_HPU_ONLY=1 uv pip install git+https://github.com/intel/auto-round.git+ echo "##[endgroup]"+}++function run_performance_test() {+ test_mode=$1+ cd /auto-round/.azure-pipelines/scripts/performance+ local log_file="perf_test_${test_mode}.log"+ rm -rf "saved" "${LOG_DIR}/${log_file}"+ echo "##[group]run ${test_mode} performance test..."+ auto-round --model_name ${model_name} --bits 4 --iters 200 --enable_torch_compile --device hpu --output_dir ./saved 2>&1 | tee -a "${LOG_DIR}/${log_file}"+ echo "##[endgroup]"+}++function main() {+ setup_environment++ install_requirements+ run_performance_test "current"++ install_baseline_requirements+ run_performance_test "baseline"++ cd /auto-round/.azure-pipelines/scripts/performance+ python check_performance.py+}++maindiff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 8ff815fef..fdcfe7085 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -1793,7 +1793,6 @@ class BaseCompressor(object):self.model = self.model.to(self.amp_dtype)layer_names = self._get_quantized_layer_names_outside_blocks()- start_time = time.time()all_first_block_names = [block[0] for block in all_blocks]if len(layer_names) > 0:logger.info(@@ -1830,6 +1829,7 @@ class BaseCompressor(object):else:pbar = tqdm(range(0, len(all_blocks[0]), self.nblocks)) # move the alg warning outside pbar+ start_time = time.time()for block_names in all_blocks:inputs = all_inputs[block_names[0]]all_inputs.pop(block_names[0])