π OpenHands: AI-Driven Development
Remove runtime unit tests (#12331)
Tim O'Farrell180df8ea206d
diff --git a/.github/workflows/ghcr-build.yml b/.github/workflows/ghcr-build.ymlindex 68462f75c2..f5eea43332 100644--- a/.github/workflows/ghcr-build.yml+++ b/.github/workflows/ghcr-build.yml@@ -252,151 +252,15 @@ jobs:-d "{\"ref\": \"main\", \"inputs\": {\"openhandsPrNumber\": \"${{ github.event.pull_request.number }}\", \"deployEnvironment\": \"feature\", \"enterpriseImageTag\": \"pr-${{ github.event.pull_request.number }}\" }}" \https://api.github.com/repos/OpenHands/deploy/actions/workflows/deploy.yaml/dispatches- # Run unit tests with the Docker runtime Docker images as root- test_runtime_root:- name: RT Unit Tests (Root)- needs: [ghcr_build_runtime, define-matrix]- runs-on: blacksmith-4vcpu-ubuntu-2404- strategy:- fail-fast: false- matrix:- base_image: ${{ fromJson(needs.define-matrix.outputs.base_image) }}- steps:- - uses: actions/checkout@v4- - name: Set up Docker Buildx- id: buildx- uses: docker/setup-buildx-action@v3- - name: Download runtime source for fork- if: github.event.pull_request.head.repo.fork- uses: actions/download-artifact@v6- with:- name: runtime-src-${{ matrix.base_image.tag }}- path: containers/runtime- - name: Lowercase Repository Owner- run: |- echo REPO_OWNER=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]') >> $GITHUB_ENV- # Forked repos can't push to GHCR, so we need to rebuild using cache- - name: Build runtime image ${{ matrix.base_image.image }} for fork- if: github.event.pull_request.head.repo.fork- uses: useblacksmith/build-push-action@v1- with:- load: true- tags: ghcr.io/${{ env.REPO_OWNER }}/runtime:${{ env.RELEVANT_SHA }}-${{ matrix.base_image.tag }}- context: containers/runtime- - name: Install poetry via pipx- run: pipx install poetry- - name: Set up Python- uses: useblacksmith/setup-python@v6- with:- python-version: "3.12"- cache: poetry- - name: Install Python dependencies using Poetry- run: make install-python-dependencies INSTALL_PLAYWRIGHT=0- - name: Run docker runtime tests- shell: bash- run: |- # We install pytest-xdist in order to run tests across CPUs- poetry run pip install pytest-xdist-- # Install to be able to retry on failures for flakey tests- poetry run pip install pytest-rerunfailures-- image_name=ghcr.io/${{ env.REPO_OWNER }}/runtime:${{ env.RELEVANT_SHA }}-${{ matrix.base_image.tag }}-- # Setting RUN_AS_OPENHANDS to false means use root.- # That should mean SANDBOX_USER_ID is ignored but some tests do not check for RUN_AS_OPENHANDS.-- TEST_RUNTIME=docker \- SANDBOX_USER_ID=$(id -u) \- SANDBOX_RUNTIME_CONTAINER_IMAGE=$image_name \- TEST_IN_CI=true \- RUN_AS_OPENHANDS=false \- poetry run pytest -n 5 -raRs --reruns 2 --reruns-delay 3 -s ./tests/runtime --ignore=tests/runtime/test_browsergym_envs.py --durations=10- env:- DEBUG: "1"-- # Run unit tests with the Docker runtime Docker images as openhands user- test_runtime_oh:- name: RT Unit Tests (openhands)- runs-on: blacksmith-4vcpu-ubuntu-2404- needs: [ghcr_build_runtime, define-matrix]- strategy:- matrix:- base_image: ${{ fromJson(needs.define-matrix.outputs.base_image) }}- steps:- - uses: actions/checkout@v4- - name: Set up Docker Buildx- id: buildx- uses: docker/setup-buildx-action@v3- - name: Download runtime source for fork- if: github.event.pull_request.head.repo.fork- uses: actions/download-artifact@v6- with:- name: runtime-src-${{ matrix.base_image.tag }}- path: containers/runtime- - name: Lowercase Repository Owner- run: |- echo REPO_OWNER=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]') >> $GITHUB_ENV- # Forked repos can't push to GHCR, so we need to rebuild using cache- - name: Build runtime image ${{ matrix.base_image.image }} for fork- if: github.event.pull_request.head.repo.fork- uses: useblacksmith/build-push-action@v1- with:- load: true- tags: ghcr.io/${{ env.REPO_OWNER }}/runtime:${{ env.RELEVANT_SHA }}-${{ matrix.base_image.tag }}- context: containers/runtime- - name: Install poetry via pipx- run: pipx install poetry- - name: Set up Python- uses: useblacksmith/setup-python@v6- with:- python-version: "3.12"- cache: poetry- - name: Install Python dependencies using Poetry- run: make install-python-dependencies POETRY_GROUP=main,test,runtime INSTALL_PLAYWRIGHT=0- - name: Run runtime tests- shell: bash- run: |- # We install pytest-xdist in order to run tests across CPUs- poetry run pip install pytest-xdist-- # Install to be able to retry on failures for flaky tests- poetry run pip install pytest-rerunfailures-- image_name=ghcr.io/${{ env.REPO_OWNER }}/runtime:${{ env.RELEVANT_SHA }}-${{ matrix.base_image.tag }}-- TEST_RUNTIME=docker \- SANDBOX_USER_ID=$(id -u) \- SANDBOX_RUNTIME_CONTAINER_IMAGE=$image_name \- TEST_IN_CI=true \- RUN_AS_OPENHANDS=true \- poetry run pytest -n 5 -raRs --reruns 2 --reruns-delay 3 -s ./tests/runtime --ignore=tests/runtime/test_browsergym_envs.py --durations=10- env:- DEBUG: "1"-- # The two following jobs (named identically) are to check whether all the runtime tests have passed as the# "All Runtime Tests Passed" is a required job for PRs to merge- # Due to this bug: https://github.com/actions/runner/issues/2566, we want to create a job that runs when the- # prerequisites have been cancelled or failed so merging is disallowed, otherwise Github considers "skipped" as "success"+ # We can remove this once the config changesruntime_tests_check_success:name: All Runtime Tests Passed- if: ${{ !cancelled() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }}runs-on: blacksmith-4vcpu-ubuntu-2204- needs: [test_runtime_root, test_runtime_oh]steps:- name: All tests passedrun: echo "All runtime tests have passed successfully!"- runtime_tests_check_fail:- name: All Runtime Tests Passed- if: ${{ cancelled() || contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}- runs-on: blacksmith-4vcpu-ubuntu-2204- needs: [test_runtime_root, test_runtime_oh]- steps:- - name: Some tests failed- run: |- echo "Some runtime tests failed or were cancelled"- exit 1update_pr_description:name: Update PR Descriptionif: github.event_name == 'pull_request' && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]'