Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Update docker user and remove baseline UT (#347)

Sun, Xuehao1cb312156163
diff --git a/.azure-pipelines/docker/Dockerfile.devel b/.azure-pipelines/docker/Dockerfile.devel
index 4e18199bf..818262ae7 100644
--- a/.azure-pipelines/docker/Dockerfile.devel
+++ b/.azure-pipelines/docker/Dockerfile.devel
@@ -37,7 +37,15 @@ RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
RUN ln -sf $(which python3) /usr/bin/python
-RUN python -m pip --no-cache-dir install --upgrade pip
+ARG USER_ID=1000
+ARG GROUP_ID=1000
+
+RUN groupadd -g ${GROUP_ID} hostgroup && \
+ useradd -m -u ${USER_ID} -g ${GROUP_ID} hostuser
+
+USER hostuser
+
+RUN python -m pip install --no-cache-dir --upgrade pip
RUN python -m pip install --no-cache-dir setuptools
RUN pip list
diff --git a/.azure-pipelines/docker/DockerfileCodeScan.devel b/.azure-pipelines/docker/DockerfileCodeScan.devel
index 110269eaf..7eba9068d 100644
--- a/.azure-pipelines/docker/DockerfileCodeScan.devel
+++ b/.azure-pipelines/docker/DockerfileCodeScan.devel
@@ -26,10 +26,20 @@ RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
python3-pip \
python3-dev \
python3-distutils \
+ autoconf \
+ build-essential \
wget
RUN ln -sf $(which python3) /usr/bin/python
+ARG USER_ID=1000
+ARG GROUP_ID=1000
+
+RUN groupadd -g ${GROUP_ID} hostgroup && \
+ useradd -m -u ${USER_ID} -g ${GROUP_ID} hostuser
+
+USER hostuser
+
RUN python -m pip install --no-cache-dir pylint==2.12.1\
bandit
diff --git a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh
index d9e2bca37..89ebd35c9 100644
--- a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh
+++ b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh
@@ -15,10 +15,6 @@ RESET="echo -en \\E[0m \\n" # close color
log_dir="/auto-round/.azure-pipelines/scripts/codeScan/scanLog"
mkdir -p $log_dir
-apt-get install -y --no-install-recommends --fix-missing \
- autoconf \
- build-essential
-
pip install -r /auto-round/requirements.txt
echo "[DEBUG] list pipdeptree..."
diff --git a/.azure-pipelines/scripts/ut/run_ut.sh b/.azure-pipelines/scripts/ut/run_ut.sh
index d9ccaebf7..411730c16 100644
--- a/.azure-pipelines/scripts/ut/run_ut.sh
+++ b/.azure-pipelines/scripts/ut/run_ut.sh
@@ -9,7 +9,7 @@ pip list
cd /auto-round/test || exit 1
find . -type f -exec sed -i '/sys\.path\.insert(0, "\.\.")/d' {} +
-export LD_LIBRARY_PATH=/usr/local/lib/:$LD_LIBRARY_PATH
+export LD_LIBRARY_PATH=${HOME}/.local/lib/:$LD_LIBRARY_PATH
export FORCE_BF16=1
export COVERAGE_RCFILE=/auto-round/.azure-pipelines/scripts/ut/.coverage
auto_round_path=$(python -c 'import auto_round; print(auto_round.__path__[0])')
diff --git a/.azure-pipelines/template/docker-template.yml b/.azure-pipelines/template/docker-template.yml
index f71123fa3..14876f176 100644
--- a/.azure-pipelines/template/docker-template.yml
+++ b/.azure-pipelines/template/docker-template.yml
@@ -62,7 +62,8 @@ steps:
- script: |
docker image prune -a -f
if [[ ! $(docker images | grep -i ${{ parameters.repoName }}:${{ parameters.repoTag }}) ]]; then
- docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/${{parameters.dockerFileName}}.devel -t ${{ parameters.repoName }}:${{ parameters.repoTag }} .
+ docker build --build-arg USER_ID=$(id -u) --build-arg GROUP_ID=$(id -g) \
+ -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/${{parameters.dockerFileName}}.devel -t ${{ parameters.repoName }}:${{ parameters.repoTag }} .
fi
docker images | grep -i ${{ parameters.repoName }}
if [[ $? -ne 0 ]]; then
diff --git a/.azure-pipelines/template/ut-template.yml b/.azure-pipelines/template/ut-template.yml
index 93f897022..f8021b62a 100644
--- a/.azure-pipelines/template/ut-template.yml
+++ b/.azure-pipelines/template/ut-template.yml
@@ -36,14 +36,14 @@ steps:
- script: |
docker exec ${{ parameters.utContainerName }} bash -c "cd /auto-round \
&& pip install -r requirements.txt \
- && pip install -vvv --no-build-isolation -e .[cpu] \
+ && pip install -vvv --no-build-isolation .[cpu] \
&& pip list"
displayName: "Env Setup"
- ${{ if eq(parameters.imageSource, 'pull') }}:
- script: |
docker exec ${{ parameters.utContainerName }} bash -c "cd /auto-round \
- && pip install -vvv --no-build-isolation -e .[hpu] \
+ && pip install -vvv --no-build-isolation .[hpu] \
&& pip list"
displayName: "HPU Env Setup"
@@ -59,6 +59,15 @@ steps:
artifact: ${{ parameters.utArtifact }}_coverage
publishLocation: "pipeline"
+ - task: UseDotNet@2
+ displayName: 'Use .NET Core sdk 7.0.x'
+ inputs:
+ version: 7.0.x
+
+ - task: PublishCodeCoverageResults@2
+ inputs:
+ summaryFileLocation: ${{ parameters.uploadPath }}/coverage.xml
+
- task: Bash@3
condition: always()
inputs:
diff --git a/.azure-pipelines/unit-test-hpu.yml b/.azure-pipelines/unit-test-hpu.yml
index 6e9220cdb..4c209bf91 100644
--- a/.azure-pipelines/unit-test-hpu.yml
+++ b/.azure-pipelines/unit-test-hpu.yml
@@ -45,18 +45,18 @@ stages:
uploadPath: $(UPLOAD_PATH)
utArtifact: "ut"
- - stage: Unit_test_baseline
- displayName: Unit Test Baseline
- dependsOn: []
- jobs:
- - job:
- displayName: Unit Test
- steps:
- - template: template/ut-template.yml
- parameters:
- imageSource: "pull"
- dockerConfigName: "gitCloneDockerConfig"
- utScriptFileName: "run_ut_hpu"
- uploadPath: $(UPLOAD_PATH)
- utArtifact: "ut_baseline"
- repo: $(REPO)
+ # - stage: Unit_test_baseline
+ # displayName: Unit Test Baseline
+ # dependsOn: []
+ # jobs:
+ # - job:
+ # displayName: Unit Test
+ # steps:
+ # - template: template/ut-template.yml
+ # parameters:
+ # imageSource: "pull"
+ # dockerConfigName: "gitCloneDockerConfig"
+ # utScriptFileName: "run_ut_hpu"
+ # uploadPath: $(UPLOAD_PATH)
+ # utArtifact: "ut_baseline"
+ # repo: $(REPO)
diff --git a/.azure-pipelines/unit-test.yml b/.azure-pipelines/unit-test.yml
index af7e64f74..ae79255c3 100644
--- a/.azure-pipelines/unit-test.yml
+++ b/.azure-pipelines/unit-test.yml
@@ -45,72 +45,72 @@ stages:
uploadPath: $(UPLOAD_PATH)
utArtifact: "ut"
- - stage: Unit_test_baseline
- displayName: Unit Test Baseline
- dependsOn: []
- jobs:
- - job:
- displayName: Unit Test
- timeoutInMinutes: 120
- steps:
- - template: template/ut-template.yml
- parameters:
- dockerConfigName: "gitCloneDockerConfig"
- utScriptFileName: "run_ut"
- uploadPath: $(UPLOAD_PATH)
- utArtifact: "ut_baseline"
- repo: $(REPO)
+ # - stage: Unit_test_baseline
+ # displayName: Unit Test Baseline
+ # dependsOn: []
+ # jobs:
+ # - job:
+ # displayName: Unit Test
+ # timeoutInMinutes: 120
+ # steps:
+ # - template: template/ut-template.yml
+ # parameters:
+ # dockerConfigName: "gitCloneDockerConfig"
+ # utScriptFileName: "run_ut"
+ # uploadPath: $(UPLOAD_PATH)
+ # utArtifact: "ut_baseline"
+ # repo: $(REPO)
- - stage: Coverage
- displayName: "Coverage Compare"
- pool:
- vmImage: "ubuntu-latest"
- dependsOn: [Unit_test, Unit_test_baseline]
- jobs:
- - job: CollectDatafiles
- steps:
- - script: |
- if [[ ! $(docker images | grep -i ${IMAGE_NAME}:${IMAGE_TAG}) ]]; then
- docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} .
- fi
- docker images | grep -i ${IMAGE_NAME}
- if [[ $? -ne 0 ]]; then
- echo "NO Such Repo"
- exit 1
- fi
- displayName: "Build develop docker image"
+ # - stage: Coverage
+ # displayName: "Coverage Compare"
+ # pool:
+ # vmImage: "ubuntu-latest"
+ # dependsOn: [Unit_test, Unit_test_baseline]
+ # jobs:
+ # - job: CollectDatafiles
+ # steps:
+ # - script: |
+ # if [[ ! $(docker images | grep -i ${IMAGE_NAME}:${IMAGE_TAG}) ]]; then
+ # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} .
+ # fi
+ # docker images | grep -i ${IMAGE_NAME}
+ # if [[ $? -ne 0 ]]; then
+ # echo "NO Such Repo"
+ # exit 1
+ # fi
+ # displayName: "Build develop docker image"
- - task: DownloadPipelineArtifact@2
- inputs:
- artifact:
- patterns: "*_coverage/.coverage"
- path: $(DOWNLOAD_PATH)
+ # - task: DownloadPipelineArtifact@2
+ # inputs:
+ # artifact:
+ # patterns: "*_coverage/.coverage"
+ # path: $(DOWNLOAD_PATH)
- - script: |
- echo "--- create container ---"
- docker run -d -it --name="collectLogs" -v ${BUILD_SOURCESDIRECTORY}:/auto-round ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash
- echo "--- docker ps ---"
- docker ps
- echo "--- collect logs ---"
- docker exec collectLogs bash -c "cd /auto-round \
- && pip install -r requirements.txt \
- && pip install -vvv --no-build-isolation -e .[cpu] \
- && pip list"
- docker exec collectLogs /bin/bash +x -c "cd /auto-round/.azure-pipelines/scripts \
- && bash ut/collect_log.sh"
- displayName: "Collect UT Coverage"
+ # - script: |
+ # echo "--- create container ---"
+ # docker run -d -it --name="collectLogs" -v ${BUILD_SOURCESDIRECTORY}:/auto-round ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash
+ # echo "--- docker ps ---"
+ # docker ps
+ # echo "--- collect logs ---"
+ # docker exec collectLogs bash -c "cd /auto-round \
+ # && pip install -r requirements.txt \
+ # && pip install -vvv --no-build-isolation -e .[cpu] \
+ # && pip list"
+ # docker exec collectLogs /bin/bash +x -c "cd /auto-round/.azure-pipelines/scripts \
+ # && bash ut/collect_log.sh"
+ # displayName: "Collect UT Coverage"
- - task: PublishPipelineArtifact@1
- condition: succeededOrFailed()
- inputs:
- targetPath: $(UPLOAD_PATH)
- artifact: $(ARTIFACT_NAME)
- publishLocation: "pipeline"
+ # - task: PublishPipelineArtifact@1
+ # condition: succeededOrFailed()
+ # inputs:
+ # targetPath: $(UPLOAD_PATH)
+ # artifact: $(ARTIFACT_NAME)
+ # publishLocation: "pipeline"
- - task: Bash@3
- condition: always()
- inputs:
- targetType: "inline"
- script: |
- docker exec collectLogs bash -c "rm -fr /auto-round/* && rm -fr /auto-round/.* || true"
- displayName: "Docker clean up"
+ # - task: Bash@3
+ # condition: always()
+ # inputs:
+ # targetType: "inline"
+ # script: |
+ # docker exec collectLogs bash -c "rm -fr /auto-round/* && rm -fr /auto-round/.* || true"
+ # displayName: "Docker clean up"