Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Enable new cpu test pool (#1435)

chen, suyue0c7f4af35f36
diff --git a/.azure-pipelines/docker/Dockerfile_xpu.devel b/.azure-pipelines/docker/Dockerfile_xpu.devel
index ca8278f41..efffa549f 100644
--- a/.azure-pipelines/docker/Dockerfile_xpu.devel
+++ b/.azure-pipelines/docker/Dockerfile_xpu.devel
@@ -28,12 +28,22 @@ RUN apt-get install -y software-properties-common \
COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
-ENV PATH="/workspace/.venv/bin:$PATH"
-ENV VIRTUAL_ENV="/workspace/.venv"
+ARG USER_ID=1000
+ARG GROUP_ID=1000
+
+RUN groupadd -g ${GROUP_ID} hostgroup && \
+ useradd -m -u ${USER_ID} -g ${GROUP_ID} hostuser && \
+ groupadd -g 991 render && \
+ usermod -aG render hostuser
+
+USER hostuser
+
+ENV PATH="/home/hostuser/.venv/bin:$PATH"
+ENV VIRTUAL_ENV="/home/hostuser/.venv"
ENV UV_NO_PROGRESS=1 \
UV_LINK_MODE=copy
-RUN uv venv --python=3.12 /workspace/.venv
+RUN uv venv --python=3.12 /home/hostuser/.venv
RUN which python && python --version
-WORKDIR /workspace
+WORKDIR /home/hostuser
diff --git a/.azure-pipelines/scripts/ut/run_ut.sh b/.azure-pipelines/scripts/ut/run_ut.sh
index 75b46e95d..eb80adff3 100644
--- a/.azure-pipelines/scripts/ut/run_ut.sh
+++ b/.azure-pipelines/scripts/ut/run_ut.sh
@@ -49,7 +49,7 @@ end_line=$(( start_line + chunk_size - 1 ))
selected_files=$(sed -n "${start_line},${end_line}p" all_tests.txt)
printf '%s\n' "${selected_files}" | sed "s,\.\/,python -m pytest --cov=\"${auto_round_path}\" --cov-report term --html=report.html --self-contained-html --cov-report xml:coverage.xml --cov-append -vs --disable-warnings ,g" > run.sh
cat run.sh
-bash run.sh 2>&1 | tee "${ut_log_name}"
+numactl --physcpubind="${NUMA_CPUSET:-0-15}" --membind="${NUMA_NODE:-0}" bash run.sh 2>&1 | tee "${ut_log_name}"
if [ $(grep -c '== FAILURES ==' ${ut_log_name}) != 0 ] || [ $(grep -c '== ERRORS ==' ${ut_log_name}) != 0 ] || [ $(grep -c 'Killed' ${ut_log_name}) != 0 ] || [ $(grep -c ' passed' ${ut_log_name}) == 0 ]; then
echo "##[error]Find errors in pytest case, please check the output..."
diff --git a/.azure-pipelines/scripts/ut/run_ut_xpu.sh b/.azure-pipelines/scripts/ut/run_ut_xpu.sh
index 195c1ebe8..61de032e8 100644
--- a/.azure-pipelines/scripts/ut/run_ut_xpu.sh
+++ b/.azure-pipelines/scripts/ut/run_ut_xpu.sh
@@ -11,7 +11,7 @@ rm -rf /auto-round/auto_round
cd /auto-round/test || exit 1
export ZE_AFFINITY_MASK=2,3 # set xpu affinity
-export LD_LIBRARY_PATH=/workspace/.venv/lib/:$LD_LIBRARY_PATH
+export LD_LIBRARY_PATH=${HOME}/.venv/lib/:$LD_LIBRARY_PATH
export COVERAGE_RCFILE=/auto-round/.azure-pipelines/scripts/ut/.coverage
auto_round_path=$(python -c 'import auto_round; print(auto_round.__path__[0])')
@@ -26,7 +26,7 @@ cat run_xpu.sh
bash run_xpu.sh 2>&1 | tee "${ut_log_name}"
-numactl -C "0-27" bash run_ark.sh 2>&1 | tee -a "${ut_log_name}"
+numactl -C "0-27" -m 0 bash run_ark.sh 2>&1 | tee -a "${ut_log_name}"
cp report.html ${LOG_DIR}/
cp coverage.xml ${LOG_DIR}/
diff --git a/.azure-pipelines/template/ut-template.yml b/.azure-pipelines/template/ut-template.yml
index 5adb1a499..954b34d5f 100644
--- a/.azure-pipelines/template/ut-template.yml
+++ b/.azure-pipelines/template/ut-template.yml
@@ -72,7 +72,8 @@ steps:
displayName: "HPU Env Setup"
- script: |
- docker exec ${{ parameters.utContainerName }} bash -c "cd /auto-round/.azure-pipelines/scripts \
+ docker exec -e NUMA_NODE=${NUMA_NODE} -e NUMA_CPUSET=${NUMA_CPUSET} ${{ parameters.utContainerName }} \
+ bash -c "cd /auto-round/.azure-pipelines/scripts \
&& bash ut/${{ parameters.utScriptFileName }}.sh ${{ parameters.utTestMode }}"
displayName: "Run UT"
diff --git a/.azure-pipelines/unit-test.yml b/.azure-pipelines/unit-test.yml
index ff08817e3..ec4e7e2e3 100644
--- a/.azure-pipelines/unit-test.yml
+++ b/.azure-pipelines/unit-test.yml
@@ -66,6 +66,7 @@ stages:
uploadPath: $(UPLOAD_PATH)
utArtifact: "ut-$(PART)"
utTestMode: $(PART)
+ utContainerName: "AutoRoundUnitTest$(NODE_LABEL)"
- stage: Coverage
displayName: "Collect Coverage"