Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

[ARK CI] Enhance auto-round-lib CI test (#1801)

chen, suyue5bbe39ba92d3
diff --git a/.azure-pipelines/ark-unit-test.yml b/.azure-pipelines/ark-unit-test.yml
new file mode 100644
index 000000000..682a20046
--- /dev/null
+++ b/.azure-pipelines/ark-unit-test.yml
@@ -0,0 +1,143 @@
+trigger: none
+
+pr:
+ autoCancel: true
+ drafts: false
+ branches:
+ include:
+ - main
+ paths:
+ include:
+ - auto_round_extension/ark/**
+ - .azure-pipelines/template/lib-build-template.yml
+ exclude:
+ - "*.md"
+ - "**/*.md"
+
+parameters:
+ - name: pythonVersions
+ displayName: "Python versions"
+ type: string
+ # Comma-separated list to avoid YAML float parsing in runtime inputs.
+ default: "3.12"
+ - name: torchVersions
+ displayName: "Torch versions"
+ type: string
+ # Comma-separated list to align with pythonVersions runtime input style.
+ default: "2.11.0"
+ - name: publishTarget
+ displayName: "Publish wheels to"
+ type: string
+ default: "none"
+ values:
+ - none
+ - testpypi
+ - pypi
+ - name: buildMode
+ displayName: "Build mode"
+ type: string
+ default: "dev"
+ values:
+ - dev
+ - release
+
+variables:
+ - name: ARTIFACT_DIR
+ value: $(Pipeline.Workspace)/auto-round-lib-wheel
+
+stages:
+ - template: template/lib-build-template.yml
+ parameters:
+ pythonVersions: ${{ parameters.pythonVersions }}
+ buildMode: ${{ parameters.buildMode }}
+
+ - stage: TestArk
+ displayName: Test Ark
+ dependsOn: BuildArkWheel
+ jobs:
+ - ${{ each pythonVersion in split(parameters.pythonVersions, ',') }}:
+ - ${{ each torchVersion in split(parameters.torchVersions, ',') }}:
+ - job: TestArk_py${{ replace(trim(pythonVersion), '.', '') }}_torch${{ replace(replace(trim(torchVersion), '.', ''), '+', '_') }}
+ displayName: "Test py${{ trim(pythonVersion) }} torch${{ trim(torchVersion) }}"
+ pool: B60
+ variables:
+ condaName: "kernel_binary_test_py${{ replace(trim(pythonVersion), '.', '') }}$(NODE_LABEL)"
+ steps:
+ - task: DownloadPipelineArtifact@2
+ inputs:
+ artifact: auto-round-lib-wheel-cp${{ replace(trim(pythonVersion), '.', '') }}
+ path: $(Build.SourcesDirectory)/ark_wheel
+
+ - task: Bash@3
+ displayName: Run tests
+ inputs:
+ targetType: inline
+ script: |
+ source ${HOME}/miniforge3/etc/profile.d/conda.sh
+ set -eo pipefail
+ cd auto_round_extension/ark
+ CONDA_ENV="$(condaName)"
+ PYTHON_VERSION="${{ trim(pythonVersion) }}"
+ TORCH_VERSION="${{ trim(torchVersion) }}"
+ if conda env list | grep -q "${CONDA_ENV}"; then conda remove -n ${CONDA_ENV} --all -y; fi
+ conda create -n ${CONDA_ENV} python=${PYTHON_VERSION} -y
+ conda activate ${CONDA_ENV}
+ conda install pip -y
+ pip install uv
+ uv pip install torch=="${TORCH_VERSION}" --index-url https://download.pytorch.org/whl/xpu
+ uv pip install $(Build.SourcesDirectory)/ark_wheel/*.whl
+ uv pip install pytest pandas
+ export LD_LIBRARY_PATH=${CONDA_PREFIX}/lib:$LD_LIBRARY_PATH
+ export ZE_AFFINITY_MASK=${ZE_AFFINITY_MASK}
+
+ echo -e "[pytest]\ntestpaths = .\n" > ./test/pytest.ini
+ cat ./test/pytest.ini
+ numactl --physcpubind="${NUMA_CPUSET:-0-27}" --membind="${NUMA_NODE:-0}" pytest -v ./test | tee kernel_ut.log 2>&1
+ if [ $(grep -c "ERROR" kernel_ut.log) != 0 ];then
+ echo "Find errors in UT test, please check the output..."
+ exit 1
+ fi
+
+ - stage: PublishWheels
+ displayName: Publish wheels (${{ parameters.publishTarget }})
+ dependsOn: TestArk
+ condition: and(succeeded(), ne('${{ parameters.publishTarget }}', 'none'))
+ jobs:
+ - job: PublishWheels
+ displayName: Publish all wheels to ${{ parameters.publishTarget }}
+ pool: B60
+ steps:
+ - checkout: none
+
+ - ${{ each pythonVersion in split(parameters.pythonVersions, ',') }}:
+ - task: DownloadPipelineArtifact@2
+ displayName: Download wheel (py${{ trim(pythonVersion) }})
+ inputs:
+ artifact: auto-round-lib-wheel-cp${{ replace(trim(pythonVersion), '.', '') }}
+ path: $(Pipeline.Workspace)/wheels
+
+ - task: Bash@3
+ displayName: Upload wheels to ${{ parameters.publishTarget }}
+ env:
+ TWINE_USERNAME: "__token__"
+ ${{ if eq(parameters.publishTarget, 'pypi') }}:
+ TWINE_PASSWORD: "$(PYPI_PASSWORD)"
+ ${{ else }}:
+ TWINE_PASSWORD: "$(TESTPYPI_PASSWORD)"
+ inputs:
+ targetType: inline
+ script: |
+ source ${HOME}/miniforge3/etc/profile.d/conda.sh
+ conda activate base
+ set -euxo pipefail
+ if [ -z "${TWINE_PASSWORD}" ]; then
+ echo "Publish password is empty, skip publishing."
+ exit 1
+ fi
+ pip install --upgrade twine
+ PUBLISH_TARGET="${{ parameters.publishTarget }}"
+ if [ "${PUBLISH_TARGET}" == "testpypi" ]; then
+ twine upload --repository-url https://test.pypi.org/legacy/ $(Pipeline.Workspace)/wheels/*.whl
+ else
+ twine upload $(Pipeline.Workspace)/wheels/*.whl
+ fi
\ No newline at end of file
diff --git a/.azure-pipelines/scripts/ut/run_ark_ut.sh b/.azure-pipelines/scripts/ut/run_ark_ut.sh
new file mode 100644
index 000000000..e99e2c883
--- /dev/null
+++ b/.azure-pipelines/scripts/ut/run_ark_ut.sh
@@ -0,0 +1,55 @@
+#!/bin/bash
+set -e
+
+source /auto-round/.azure-pipelines/scripts/change_color.sh
+
+function setup_environment() {
+ echo "##[group]set up ARK UT env..."
+ cd /auto-round/auto_round_extension/ark
+ uv pip install -r requirements.txt
+ uv pip install /auto-round/ark_wheel/*.whl --no-deps
+ uv pip install pytest pandas
+ uv pip list
+ echo "##[endgroup]"
+
+ LOG_DIR=/auto-round/log_dir
+ mkdir -p ${LOG_DIR}
+ SUMMARY_LOG="${LOG_DIR}/results_summary.log"
+}
+
+function run_unit_test() {
+ cd /auto-round/auto_round_extension/ark
+
+ for test_file in $(find ./test -name "test*.py" | sort); do
+ local test_basename=$(basename ${test_file} .py)
+
+ echo "##[group]Running ark ${test_file}..."
+ local ut_log_name="${LOG_DIR}/unittest_ark_${test_basename}.log"
+ pytest -v ${test_file} 2>&1 | tee ${ut_log_name}
+ echo "##[endgroup]"
+ done
+}
+
+function print_summary() {
+ local status=0
+ for log_file in ${LOG_DIR}/unittest_*.log; do
+ if grep -q "FAILED" "${log_file}"; then
+ $LIGHT_RED && echo "FAILED: ${log_file}" && $RESET
+ status=1
+ elif grep -q "ERROR" "${log_file}"; then
+ $LIGHT_RED && echo "ERROR: ${log_file}" && $RESET
+ status=1
+ elif grep -q "passed" "${log_file}"; then
+ $LIGHT_GREEN && echo "PASSED: ${log_file}" && $RESET
+ fi
+ done
+ exit $status
+}
+
+function main() {
+ setup_environment
+ run_unit_test
+ print_summary
+}
+
+main
diff --git a/.azure-pipelines/template/lib-build-template.yml b/.azure-pipelines/template/lib-build-template.yml
index dcf6ec204..1f18b68ed 100644
--- a/.azure-pipelines/template/lib-build-template.yml
+++ b/.azure-pipelines/template/lib-build-template.yml
@@ -5,6 +5,13 @@ parameters:
- name: publishToTestPyPI
type: boolean
default: false
+ - name: pythonVersions
+ type: string
+ # Comma-separated list to avoid YAML float parsing (e.g. 3.10 -> 3.1).
+ default: "3.12"
+ - name: buildMode
+ type: string
+ default: "dev"
stages:
- ${{ if eq(parameters.enableChangeDetection, true) }}:
@@ -61,66 +68,69 @@ stages:
variables:
BUILD_ARK_WHEEL: $[ stageDependencies.DetectArkChanges.Detect.outputs['SetArkWheelFlag.BUILD_ARK_WHEEL'] ]
jobs:
- - job: BuildArkWheel
- displayName: Build ark wheel in manylinux
- ${{ if eq(parameters.enableChangeDetection, true) }}:
- condition: and(succeeded(), eq(variables['BUILD_ARK_WHEEL'], 'true'))
- pool: LIB-BUILD-POOL
- timeoutInMinutes: 10
- steps:
- - checkout: self
- clean: true
- fetchTags: true
-
- - task: Bash@3
- displayName: Build wheel in manylinux container
- inputs:
- targetType: inline
- script: |
- set -euxo pipefail
- docker run --rm \
- --user "$(id -u):$(id -g)" \
- -e HOME=/tmp \
- -v $(Build.SourcesDirectory):/auto-round \
- -w /auto-round/auto_round_extension/ark \
- pytorch/manylinux2_28-builder:xpu-v2.10.0-rc7 \
- bash -c " /opt/python/cp312-cp312/bin/python -m venv venv \
- && source venv/bin/activate \
- && source /opt/intel/oneapi/setvars.sh \
- && pip install -U pip build cmake setuptools \
- && python -m build --wheel --no-isolation \
- && auditwheel repair dist/*.whl --exclude lib* -w wheelhouse/ \
- && ls -lah wheelhouse"
-
- - task: PublishPipelineArtifact@1
- displayName: Publish wheel artifact
- inputs:
- targetPath: $(Build.SourcesDirectory)/auto_round_extension/ark/wheelhouse
- artifact: auto-round-lib-wheel
- publishLocation: pipeline
+ - ${{ each pythonVersion in split(parameters.pythonVersions, ',') }}:
+ - job: BuildArkWheel_cp${{ replace(trim(pythonVersion), '.', '') }}
+ displayName: Build ark wheel (py${{ trim(pythonVersion) }})
+ ${{ if eq(parameters.enableChangeDetection, true) }}:
+ condition: and(succeeded(), eq(variables['BUILD_ARK_WHEEL'], 'true'))
+ pool: GAUDI
+ timeoutInMinutes: 30
+ steps:
+ - checkout: self
+ clean: true
- - ${{ if eq(parameters.publishToTestPyPI, true) }}:
- task: Bash@3
- displayName: Publish wheel to TestPyPI
- condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main'), or(eq(variables['Build.Reason'], 'IndividualCI'), eq(variables['Build.Reason'], 'BatchedCI')))
- env:
- TWINE_USERNAME: "__token__"
- TWINE_PASSWORD: "$(TESTPYPI_PASSWORD)"
+ displayName: Build wheel in manylinux container
inputs:
targetType: inline
script: |
set -euxo pipefail
- if [ -z "${TWINE_PASSWORD}" ]; then
- echo "TESTPYPI_PASSWORD is empty, skip publishing."
- exit 1
- fi
- docker run --rm --privileged \
+ python_version_fix=${{ replace(trim(pythonVersion), '.', '') }}
+ docker run --rm \
--user "$(id -u):$(id -g)" \
-e HOME=/tmp \
-v $(Build.SourcesDirectory):/auto-round \
+ -v /opt/intel/oneapi:/opt/intel/oneapi \
-w /auto-round/auto_round_extension/ark \
- -e TWINE_USERNAME \
- -e TWINE_PASSWORD \
- pytorch/manylinux2_28-builder:xpu-v2.10.0-rc7 \
- bash -c "python -m pip install --upgrade twine \
- && twine upload --repository-url https://test.pypi.org/legacy/ wheelhouse/*.whl"
\ No newline at end of file
+ quay.io/pypa/manylinux_2_28_x86_64 \
+ bash -c " /opt/python/cp${python_version_fix}-cp${python_version_fix}/bin/python -m venv venv \
+ && source venv/bin/activate \
+ && source /opt/intel/oneapi/setvars.sh \
+ && pip install -U pip build cmake setuptools \
+ && export BUILD_MODE=${{ parameters.buildMode }} \
+ && python -m build --wheel --no-isolation \
+ && auditwheel repair dist/*.whl --exclude lib* -w wheelhouse/ \
+ && ls -lah wheelhouse"
+
+ - task: PublishPipelineArtifact@1
+ displayName: Publish wheel artifact
+ inputs:
+ targetPath: $(Build.SourcesDirectory)/auto_round_extension/ark/wheelhouse
+ artifact: auto-round-lib-wheel-cp${{ replace(trim(pythonVersion), '.', '') }}
+ publishLocation: pipeline
+
+ - ${{ if eq(parameters.publishToTestPyPI, true) }}:
+ - task: Bash@3
+ displayName: Publish wheel to TestPyPI
+ env:
+ TWINE_USERNAME: "__token__"
+ TWINE_PASSWORD: "$(TESTPYPI_PASSWORD)"
+ inputs:
+ targetType: inline
+ script: |
+ set -euxo pipefail
+ if [ -z "${TWINE_PASSWORD}" ]; then
+ echo "TESTPYPI_PASSWORD is empty, skip publishing."
+ exit 1
+ fi
+ docker run --rm --privileged \
+ --user "$(id -u):$(id -g)" \
+ -e HOME=/tmp \
+ -v $(Build.SourcesDirectory):/auto-round \
+ -w /auto-round/auto_round_extension/ark \
+ -e TWINE_USERNAME \
+ -e TWINE_PASSWORD \
+ quay.io/pypa/manylinux_2_28_x86_64 \
+ bash -c "source venv/bin/activate \
+ && python -m pip install --upgrade twine \
+ && twine upload --repository-url https://test.pypi.org/legacy/ wheelhouse/*.whl"
\ No newline at end of file
diff --git a/.azure-pipelines/template/ut-template.yml b/.azure-pipelines/template/ut-template.yml
index 2066a8802..2b6efbe2a 100644
--- a/.azure-pipelines/template/ut-template.yml
+++ b/.azure-pipelines/template/ut-template.yml
@@ -48,7 +48,7 @@ steps:
- task: DownloadPipelineArtifact@2
condition: and(succeeded(), eq(variables['BUILD_ARK_WHEEL'], 'true'))
inputs:
- artifact: auto-round-lib-wheel
+ artifact: auto-round-lib-wheel-cp312
patterns: '*.whl'
path: $(Build.SourcesDirectory)/ark_wheel
displayName: "Download ark wheel"
diff --git a/.azure-pipelines/unit-test-cuda.yml b/.azure-pipelines/unit-test-cuda.yml
index 410c2c8f5..83deed2fb 100644
--- a/.azure-pipelines/unit-test-cuda.yml
+++ b/.azure-pipelines/unit-test-cuda.yml
@@ -19,6 +19,7 @@ pr:
exclude:
- "*.md"
- "**/*.md"
+ - auto_round_extension/ark
variables:
- name: POOL_NAME
diff --git a/auto_round_extension/ark/setup.py b/auto_round_extension/ark/setup.py
index f862c0e3a..fb957b4ac 100644
--- a/auto_round_extension/ark/setup.py
+++ b/auto_round_extension/ark/setup.py
@@ -22,20 +22,26 @@ except Exception as error:
def get_build_version():
+ print(f"Getting build version for build mode: {build_mode}")
+ if build_mode == "release":
+ return __version__
if os.path.exists("PKG-INFO"):
with open("PKG-INFO", encoding="utf-8") as f:
for line in f:
if line.startswith("Version:"):
- return line.split(":", 1)[1].strip()
-
- if build_mode == "release":
- return __version__
+ version = line.split(":", 1)[1].strip()
+ print(f"Read version from PKG-INFO: {version}")
+ return version
try:
- result = subprocess.run(["git", "describe", "--tags"], capture_output=True, text=True, check=True)
- distance = result.stdout.strip().split("-")[-2]
- commit = result.stdout.strip().split("-")[-1]
- return f"{__version__}.dev{distance}+{commit}"
- except subprocess.CalledProcessError:
+ result = subprocess.run(
+ ["git", "log", "-1", "--format=%cd", "--date=format:%Y%m%d%H%M"], capture_output=True, text=True, check=True
+ )
+ date_str = result.stdout.strip()
+ version = f"{__version__}.dev{date_str}"
+ print(f"Git commit date: {date_str}, version: {version}")
+ return version
+ except subprocess.CalledProcessError as e:
+ print(f"Warning: git command failed: {e}. Falling back to version {__version__}")
return __version__