PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Update docker user and remove baseline UT (#347)
Sun, Xuehao1cb312156163
diff --git a/.azure-pipelines/docker/Dockerfile.devel b/.azure-pipelines/docker/Dockerfile.develindex 4e18199bf..818262ae7 100644--- a/.azure-pipelines/docker/Dockerfile.devel+++ b/.azure-pipelines/docker/Dockerfile.devel@@ -37,7 +37,15 @@ RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \RUN ln -sf $(which python3) /usr/bin/python-RUN python -m pip --no-cache-dir install --upgrade pip+ARG USER_ID=1000+ARG GROUP_ID=1000++RUN groupadd -g ${GROUP_ID} hostgroup && \+ useradd -m -u ${USER_ID} -g ${GROUP_ID} hostuser++USER hostuser++RUN python -m pip install --no-cache-dir --upgrade pipRUN python -m pip install --no-cache-dir setuptoolsRUN pip listdiff --git a/.azure-pipelines/docker/DockerfileCodeScan.devel b/.azure-pipelines/docker/DockerfileCodeScan.develindex 110269eaf..7eba9068d 100644--- a/.azure-pipelines/docker/DockerfileCodeScan.devel+++ b/.azure-pipelines/docker/DockerfileCodeScan.devel@@ -26,10 +26,20 @@ RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \python3-pip \python3-dev \python3-distutils \+ autoconf \+ build-essential \wgetRUN ln -sf $(which python3) /usr/bin/python+ARG USER_ID=1000+ARG GROUP_ID=1000++RUN groupadd -g ${GROUP_ID} hostgroup && \+ useradd -m -u ${USER_ID} -g ${GROUP_ID} hostuser++USER hostuser+RUN python -m pip install --no-cache-dir pylint==2.12.1\banditdiff --git a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh b/.azure-pipelines/scripts/codeScan/pylint/pylint.shindex d9e2bca37..89ebd35c9 100644--- a/.azure-pipelines/scripts/codeScan/pylint/pylint.sh+++ b/.azure-pipelines/scripts/codeScan/pylint/pylint.sh@@ -15,10 +15,6 @@ RESET="echo -en \\E[0m \\n" # close colorlog_dir="/auto-round/.azure-pipelines/scripts/codeScan/scanLog"mkdir -p $log_dir-apt-get install -y --no-install-recommends --fix-missing \- autoconf \- build-essential-pip install -r /auto-round/requirements.txtecho "[DEBUG] list pipdeptree..."diff --git a/.azure-pipelines/scripts/ut/run_ut.sh b/.azure-pipelines/scripts/ut/run_ut.shindex d9ccaebf7..411730c16 100644--- a/.azure-pipelines/scripts/ut/run_ut.sh+++ b/.azure-pipelines/scripts/ut/run_ut.sh@@ -9,7 +9,7 @@ pip listcd /auto-round/test || exit 1find . -type f -exec sed -i '/sys\.path\.insert(0, "\.\.")/d' {} +-export LD_LIBRARY_PATH=/usr/local/lib/:$LD_LIBRARY_PATH+export LD_LIBRARY_PATH=${HOME}/.local/lib/:$LD_LIBRARY_PATHexport FORCE_BF16=1export COVERAGE_RCFILE=/auto-round/.azure-pipelines/scripts/ut/.coverageauto_round_path=$(python -c 'import auto_round; print(auto_round.__path__[0])')diff --git a/.azure-pipelines/template/docker-template.yml b/.azure-pipelines/template/docker-template.ymlindex f71123fa3..14876f176 100644--- a/.azure-pipelines/template/docker-template.yml+++ b/.azure-pipelines/template/docker-template.yml@@ -62,7 +62,8 @@ steps:- script: |docker image prune -a -fif [[ ! $(docker images | grep -i ${{ parameters.repoName }}:${{ parameters.repoTag }}) ]]; then- docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/${{parameters.dockerFileName}}.devel -t ${{ parameters.repoName }}:${{ parameters.repoTag }} .+ docker build --build-arg USER_ID=$(id -u) --build-arg GROUP_ID=$(id -g) \+ -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/${{parameters.dockerFileName}}.devel -t ${{ parameters.repoName }}:${{ parameters.repoTag }} .fidocker images | grep -i ${{ parameters.repoName }}if [[ $? -ne 0 ]]; thendiff --git a/.azure-pipelines/template/ut-template.yml b/.azure-pipelines/template/ut-template.ymlindex 93f897022..f8021b62a 100644--- a/.azure-pipelines/template/ut-template.yml+++ b/.azure-pipelines/template/ut-template.yml@@ -36,14 +36,14 @@ steps:- script: |docker exec ${{ parameters.utContainerName }} bash -c "cd /auto-round \&& pip install -r requirements.txt \- && pip install -vvv --no-build-isolation -e .[cpu] \+ && pip install -vvv --no-build-isolation .[cpu] \&& pip list"displayName: "Env Setup"- ${{ if eq(parameters.imageSource, 'pull') }}:- script: |docker exec ${{ parameters.utContainerName }} bash -c "cd /auto-round \- && pip install -vvv --no-build-isolation -e .[hpu] \+ && pip install -vvv --no-build-isolation .[hpu] \&& pip list"displayName: "HPU Env Setup"@@ -59,6 +59,15 @@ steps:artifact: ${{ parameters.utArtifact }}_coveragepublishLocation: "pipeline"+ - task: UseDotNet@2+ displayName: 'Use .NET Core sdk 7.0.x'+ inputs:+ version: 7.0.x++ - task: PublishCodeCoverageResults@2+ inputs:+ summaryFileLocation: ${{ parameters.uploadPath }}/coverage.xml+- task: Bash@3condition: always()inputs:diff --git a/.azure-pipelines/unit-test-hpu.yml b/.azure-pipelines/unit-test-hpu.ymlindex 6e9220cdb..4c209bf91 100644--- a/.azure-pipelines/unit-test-hpu.yml+++ b/.azure-pipelines/unit-test-hpu.yml@@ -45,18 +45,18 @@ stages:uploadPath: $(UPLOAD_PATH)utArtifact: "ut"- - stage: Unit_test_baseline- displayName: Unit Test Baseline- dependsOn: []- jobs:- - job:- displayName: Unit Test- steps:- - template: template/ut-template.yml- parameters:- imageSource: "pull"- dockerConfigName: "gitCloneDockerConfig"- utScriptFileName: "run_ut_hpu"- uploadPath: $(UPLOAD_PATH)- utArtifact: "ut_baseline"- repo: $(REPO)+ # - stage: Unit_test_baseline+ # displayName: Unit Test Baseline+ # dependsOn: []+ # jobs:+ # - job:+ # displayName: Unit Test+ # steps:+ # - template: template/ut-template.yml+ # parameters:+ # imageSource: "pull"+ # dockerConfigName: "gitCloneDockerConfig"+ # utScriptFileName: "run_ut_hpu"+ # uploadPath: $(UPLOAD_PATH)+ # utArtifact: "ut_baseline"+ # repo: $(REPO)diff --git a/.azure-pipelines/unit-test.yml b/.azure-pipelines/unit-test.ymlindex af7e64f74..ae79255c3 100644--- a/.azure-pipelines/unit-test.yml+++ b/.azure-pipelines/unit-test.yml@@ -45,72 +45,72 @@ stages:uploadPath: $(UPLOAD_PATH)utArtifact: "ut"- - stage: Unit_test_baseline- displayName: Unit Test Baseline- dependsOn: []- jobs:- - job:- displayName: Unit Test- timeoutInMinutes: 120- steps:- - template: template/ut-template.yml- parameters:- dockerConfigName: "gitCloneDockerConfig"- utScriptFileName: "run_ut"- uploadPath: $(UPLOAD_PATH)- utArtifact: "ut_baseline"- repo: $(REPO)+ # - stage: Unit_test_baseline+ # displayName: Unit Test Baseline+ # dependsOn: []+ # jobs:+ # - job:+ # displayName: Unit Test+ # timeoutInMinutes: 120+ # steps:+ # - template: template/ut-template.yml+ # parameters:+ # dockerConfigName: "gitCloneDockerConfig"+ # utScriptFileName: "run_ut"+ # uploadPath: $(UPLOAD_PATH)+ # utArtifact: "ut_baseline"+ # repo: $(REPO)- - stage: Coverage- displayName: "Coverage Compare"- pool:- vmImage: "ubuntu-latest"- dependsOn: [Unit_test, Unit_test_baseline]- jobs:- - job: CollectDatafiles- steps:- - script: |- if [[ ! $(docker images | grep -i ${IMAGE_NAME}:${IMAGE_TAG}) ]]; then- docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} .- fi- docker images | grep -i ${IMAGE_NAME}- if [[ $? -ne 0 ]]; then- echo "NO Such Repo"- exit 1- fi- displayName: "Build develop docker image"+ # - stage: Coverage+ # displayName: "Coverage Compare"+ # pool:+ # vmImage: "ubuntu-latest"+ # dependsOn: [Unit_test, Unit_test_baseline]+ # jobs:+ # - job: CollectDatafiles+ # steps:+ # - script: |+ # if [[ ! $(docker images | grep -i ${IMAGE_NAME}:${IMAGE_TAG}) ]]; then+ # docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} .+ # fi+ # docker images | grep -i ${IMAGE_NAME}+ # if [[ $? -ne 0 ]]; then+ # echo "NO Such Repo"+ # exit 1+ # fi+ # displayName: "Build develop docker image"- - task: DownloadPipelineArtifact@2- inputs:- artifact:- patterns: "*_coverage/.coverage"- path: $(DOWNLOAD_PATH)+ # - task: DownloadPipelineArtifact@2+ # inputs:+ # artifact:+ # patterns: "*_coverage/.coverage"+ # path: $(DOWNLOAD_PATH)- - script: |- echo "--- create container ---"- docker run -d -it --name="collectLogs" -v ${BUILD_SOURCESDIRECTORY}:/auto-round ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash- echo "--- docker ps ---"- docker ps- echo "--- collect logs ---"- docker exec collectLogs bash -c "cd /auto-round \- && pip install -r requirements.txt \- && pip install -vvv --no-build-isolation -e .[cpu] \- && pip list"- docker exec collectLogs /bin/bash +x -c "cd /auto-round/.azure-pipelines/scripts \- && bash ut/collect_log.sh"- displayName: "Collect UT Coverage"+ # - script: |+ # echo "--- create container ---"+ # docker run -d -it --name="collectLogs" -v ${BUILD_SOURCESDIRECTORY}:/auto-round ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash+ # echo "--- docker ps ---"+ # docker ps+ # echo "--- collect logs ---"+ # docker exec collectLogs bash -c "cd /auto-round \+ # && pip install -r requirements.txt \+ # && pip install -vvv --no-build-isolation -e .[cpu] \+ # && pip list"+ # docker exec collectLogs /bin/bash +x -c "cd /auto-round/.azure-pipelines/scripts \+ # && bash ut/collect_log.sh"+ # displayName: "Collect UT Coverage"- - task: PublishPipelineArtifact@1- condition: succeededOrFailed()- inputs:- targetPath: $(UPLOAD_PATH)- artifact: $(ARTIFACT_NAME)- publishLocation: "pipeline"+ # - task: PublishPipelineArtifact@1+ # condition: succeededOrFailed()+ # inputs:+ # targetPath: $(UPLOAD_PATH)+ # artifact: $(ARTIFACT_NAME)+ # publishLocation: "pipeline"- - task: Bash@3- condition: always()- inputs:- targetType: "inline"- script: |- docker exec collectLogs bash -c "rm -fr /auto-round/* && rm -fr /auto-round/.* || true"- displayName: "Docker clean up"+ # - task: Bash@3+ # condition: always()+ # inputs:+ # targetType: "inline"+ # script: |+ # docker exec collectLogs bash -c "rm -fr /auto-round/* && rm -fr /auto-round/.* || true"+ # displayName: "Docker clean up"