Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Add Performance Test (#1561)

Sun, Xuehao29f0275126cd
diff --git a/.azure-pipelines/performance-test.yaml b/.azure-pipelines/performance-test.yaml
new file mode 100644
index 000000000..b38a4e0af
--- /dev/null
+++ b/.azure-pipelines/performance-test.yaml
@@ -0,0 +1,67 @@
+trigger: none
+
+pr:
+ autoCancel: true
+ drafts: false
+ branches:
+ include:
+ - main
+ paths:
+ include:
+ - auto_round
+ - auto_round_extension
+ - .azure-pipelines/scripts/performance
+ - .azure-pipelines/performance-test.yaml
+ exclude:
+ - "*.md"
+ - "**/*.md"
+
+variables:
+ CONTAINER_NAME: "AutoRoundPerfTest"
+
+pool: GAUDI
+
+stages:
+ - stage: PerformanceTest
+ displayName: Performance Test
+ dependsOn: []
+ jobs:
+ - job: PerformanceTestJob
+ displayName: Performance Test Job
+ steps:
+ - template: template/docker-template.yml
+ parameters:
+ containerName: $(CONTAINER_NAME)
+ imageSource: pull
+
+ - script: |
+ docker exec $(CONTAINER_NAME) bash -c "cd /auto-round \
+ && export UV_NO_PROGRESS=1 \
+ && export UV_SYSTEM_PYTHON=1 \
+ && pip install uv \
+ && uv pip install -r test/test_hpu/requirements.txt \
+ && uv pip install huggingface_hub \
+ && uv pip list"
+ displayName: "HPU Env Setup"
+
+ - script: |
+ docker exec $(CONTAINER_NAME) \
+ bash -c "cd /auto-round/.azure-pipelines/scripts/performance \
+ && bash perf_test.sh"
+ displayName: "Run performance test"
+
+ - task: PublishPipelineArtifact@1
+ condition: succeededOrFailed()
+ inputs:
+ targetPath: $(Build.SourcesDirectory)/log_dir
+ artifact: "Performance_Test_Logs"
+ publishLocation: "pipeline"
+
+ - task: Bash@3
+ condition: always()
+ inputs:
+ targetType: "inline"
+ script: |
+ docker stop $(CONTAINER_NAME)
+ docker rm -vf $(CONTAINER_NAME) || true
+ displayName: "Docker clean up"
diff --git a/.azure-pipelines/scripts/performance/check_performance.py b/.azure-pipelines/scripts/performance/check_performance.py
new file mode 100644
index 000000000..079b3b0a7
--- /dev/null
+++ b/.azure-pipelines/scripts/performance/check_performance.py
@@ -0,0 +1,71 @@
+import re
+import sys
+
+LOG_DIR = "/auto-round/log_dir"
+
+
+def parse_tuning_time(log_file):
+ with open(log_file, "r") as f:
+ content = f.read()
+
+ pattern = r"tuning time ([0-9]+\.[0-9]+)"
+ match = re.search(pattern, content)
+
+ if match:
+ elapsed = str_to_float(match.group(1))
+ return elapsed
+ return None
+
+
+def str_to_float(value):
+ try:
+ return round(float(value), 4)
+ except ValueError:
+ return value
+
+
+def get_tuning_time():
+ summary = {}
+ model_list = ["Qwen/Qwen3-0.6B"]
+ for model in model_list:
+ summary[model] = {}
+ for test_mode in ["current", "baseline"]:
+ log_file = f"{LOG_DIR}/perf_test_{test_mode}.log"
+ print(f"Processing {log_file}...")
+ tuning_time = parse_tuning_time(log_file)
+ if tuning_time is not None:
+ summary[model][test_mode] = tuning_time
+ else:
+ summary[model][test_mode] = "N/A"
+
+ return summary
+
+
+def check_performance():
+ status = True
+ summary = get_tuning_time()
+ for model, times in summary.items():
+ current_time = times.get("current", "N/A")
+ baseline_time = times.get("baseline", "N/A")
+ if current_time != "N/A" and baseline_time != "N/A":
+ print(f"{model}:\n Current = {current_time} seconds\n Baseline = {baseline_time} seconds")
+ ratio = current_time / baseline_time
+ if ratio < 0.9 or ratio > 1.1:
+ status = False
+ else:
+ print(f"{model}: Tuning time data is incomplete.")
+ status = False
+
+ if status:
+ print("Performance check passed: Current tuning times are within acceptable limits compared to baseline.")
+ else:
+ print("Performance check failed: Current tuning times exceed acceptable limits compared to baseline.")
+ sys.exit(1)
+
+
+def main():
+ check_performance()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/.azure-pipelines/scripts/performance/perf_test.sh b/.azure-pipelines/scripts/performance/perf_test.sh
new file mode 100644
index 000000000..f24c1aac6
--- /dev/null
+++ b/.azure-pipelines/scripts/performance/perf_test.sh
@@ -0,0 +1,59 @@
+#!/bin/bash
+set -e
+
+function setup_environment() {
+ echo "##[group]set up environment..."
+ export TZ='Asia/Shanghai'
+ export TQDM_MININTERVAL=60
+ export HF_HUB_DISABLE_PROGRESS_BARS=1
+ export UV_NO_PROGRESS=1
+ export UV_SYSTEM_PYTHON=1
+
+ LOG_DIR="/auto-round/log_dir"
+ mkdir -p "${LOG_DIR}"
+
+ model_name="Qwen/Qwen3-0.6B"
+ hf download ${model_name}
+ echo "##[endgroup]"
+}
+
+function install_requirements() {
+ echo "##[group]install requirements..."
+ cd /auto-round
+ uv pip uninstall auto-round || true
+ BUILD_HPU_ONLY=1 uv pip install .
+ echo "##[endgroup]"
+}
+
+function install_baseline_requirements() {
+ echo "##[group]install baseline requirements..."
+ cd /auto-round
+ uv pip uninstall auto-round || true
+ BUILD_HPU_ONLY=1 uv pip install git+https://github.com/intel/auto-round.git
+ echo "##[endgroup]"
+}
+
+function run_performance_test() {
+ test_mode=$1
+ cd /auto-round/.azure-pipelines/scripts/performance
+ local log_file="perf_test_${test_mode}.log"
+ rm -rf "saved" "${LOG_DIR}/${log_file}"
+ echo "##[group]run ${test_mode} performance test..."
+ auto-round --model_name ${model_name} --bits 4 --iters 200 --enable_torch_compile --device hpu --output_dir ./saved 2>&1 | tee -a "${LOG_DIR}/${log_file}"
+ echo "##[endgroup]"
+}
+
+function main() {
+ setup_environment
+
+ install_requirements
+ run_performance_test "current"
+
+ install_baseline_requirements
+ run_performance_test "baseline"
+
+ cd /auto-round/.azure-pipelines/scripts/performance
+ python check_performance.py
+}
+
+main
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index 8ff815fef..fdcfe7085 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -1793,7 +1793,6 @@ class BaseCompressor(object):
self.model = self.model.to(self.amp_dtype)
layer_names = self._get_quantized_layer_names_outside_blocks()
- start_time = time.time()
all_first_block_names = [block[0] for block in all_blocks]
if len(layer_names) > 0:
logger.info(
@@ -1830,6 +1829,7 @@ class BaseCompressor(object):
else:
pbar = tqdm(range(0, len(all_blocks[0]), self.nblocks)) # move the alg warning outside pbar
+ start_time = time.time()
for block_names in all_blocks:
inputs = all_inputs[block_names[0]]
all_inputs.pop(block_names[0])