Skip to content

πŸ™Œ OpenHands: AI-Driven Development

Remove runtime unit tests (#12331)

Tim O'Farrell180df8ea206d
diff --git a/.github/workflows/ghcr-build.yml b/.github/workflows/ghcr-build.yml
index 68462f75c2..f5eea43332 100644
--- a/.github/workflows/ghcr-build.yml
+++ b/.github/workflows/ghcr-build.yml
@@ -252,151 +252,15 @@ jobs:
-d "{\"ref\": \"main\", \"inputs\": {\"openhandsPrNumber\": \"${{ github.event.pull_request.number }}\", \"deployEnvironment\": \"feature\", \"enterpriseImageTag\": \"pr-${{ github.event.pull_request.number }}\" }}" \
https://api.github.com/repos/OpenHands/deploy/actions/workflows/deploy.yaml/dispatches
- # Run unit tests with the Docker runtime Docker images as root
- test_runtime_root:
- name: RT Unit Tests (Root)
- needs: [ghcr_build_runtime, define-matrix]
- runs-on: blacksmith-4vcpu-ubuntu-2404
- strategy:
- fail-fast: false
- matrix:
- base_image: ${{ fromJson(needs.define-matrix.outputs.base_image) }}
- steps:
- - uses: actions/checkout@v4
- - name: Set up Docker Buildx
- id: buildx
- uses: docker/setup-buildx-action@v3
- - name: Download runtime source for fork
- if: github.event.pull_request.head.repo.fork
- uses: actions/download-artifact@v6
- with:
- name: runtime-src-${{ matrix.base_image.tag }}
- path: containers/runtime
- - name: Lowercase Repository Owner
- run: |
- echo REPO_OWNER=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]') >> $GITHUB_ENV
- # Forked repos can't push to GHCR, so we need to rebuild using cache
- - name: Build runtime image ${{ matrix.base_image.image }} for fork
- if: github.event.pull_request.head.repo.fork
- uses: useblacksmith/build-push-action@v1
- with:
- load: true
- tags: ghcr.io/${{ env.REPO_OWNER }}/runtime:${{ env.RELEVANT_SHA }}-${{ matrix.base_image.tag }}
- context: containers/runtime
- - name: Install poetry via pipx
- run: pipx install poetry
- - name: Set up Python
- uses: useblacksmith/setup-python@v6
- with:
- python-version: "3.12"
- cache: poetry
- - name: Install Python dependencies using Poetry
- run: make install-python-dependencies INSTALL_PLAYWRIGHT=0
- - name: Run docker runtime tests
- shell: bash
- run: |
- # We install pytest-xdist in order to run tests across CPUs
- poetry run pip install pytest-xdist
-
- # Install to be able to retry on failures for flakey tests
- poetry run pip install pytest-rerunfailures
-
- image_name=ghcr.io/${{ env.REPO_OWNER }}/runtime:${{ env.RELEVANT_SHA }}-${{ matrix.base_image.tag }}
-
- # Setting RUN_AS_OPENHANDS to false means use root.
- # That should mean SANDBOX_USER_ID is ignored but some tests do not check for RUN_AS_OPENHANDS.
-
- TEST_RUNTIME=docker \
- SANDBOX_USER_ID=$(id -u) \
- SANDBOX_RUNTIME_CONTAINER_IMAGE=$image_name \
- TEST_IN_CI=true \
- RUN_AS_OPENHANDS=false \
- poetry run pytest -n 5 -raRs --reruns 2 --reruns-delay 3 -s ./tests/runtime --ignore=tests/runtime/test_browsergym_envs.py --durations=10
- env:
- DEBUG: "1"
-
- # Run unit tests with the Docker runtime Docker images as openhands user
- test_runtime_oh:
- name: RT Unit Tests (openhands)
- runs-on: blacksmith-4vcpu-ubuntu-2404
- needs: [ghcr_build_runtime, define-matrix]
- strategy:
- matrix:
- base_image: ${{ fromJson(needs.define-matrix.outputs.base_image) }}
- steps:
- - uses: actions/checkout@v4
- - name: Set up Docker Buildx
- id: buildx
- uses: docker/setup-buildx-action@v3
- - name: Download runtime source for fork
- if: github.event.pull_request.head.repo.fork
- uses: actions/download-artifact@v6
- with:
- name: runtime-src-${{ matrix.base_image.tag }}
- path: containers/runtime
- - name: Lowercase Repository Owner
- run: |
- echo REPO_OWNER=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]') >> $GITHUB_ENV
- # Forked repos can't push to GHCR, so we need to rebuild using cache
- - name: Build runtime image ${{ matrix.base_image.image }} for fork
- if: github.event.pull_request.head.repo.fork
- uses: useblacksmith/build-push-action@v1
- with:
- load: true
- tags: ghcr.io/${{ env.REPO_OWNER }}/runtime:${{ env.RELEVANT_SHA }}-${{ matrix.base_image.tag }}
- context: containers/runtime
- - name: Install poetry via pipx
- run: pipx install poetry
- - name: Set up Python
- uses: useblacksmith/setup-python@v6
- with:
- python-version: "3.12"
- cache: poetry
- - name: Install Python dependencies using Poetry
- run: make install-python-dependencies POETRY_GROUP=main,test,runtime INSTALL_PLAYWRIGHT=0
- - name: Run runtime tests
- shell: bash
- run: |
- # We install pytest-xdist in order to run tests across CPUs
- poetry run pip install pytest-xdist
-
- # Install to be able to retry on failures for flaky tests
- poetry run pip install pytest-rerunfailures
-
- image_name=ghcr.io/${{ env.REPO_OWNER }}/runtime:${{ env.RELEVANT_SHA }}-${{ matrix.base_image.tag }}
-
- TEST_RUNTIME=docker \
- SANDBOX_USER_ID=$(id -u) \
- SANDBOX_RUNTIME_CONTAINER_IMAGE=$image_name \
- TEST_IN_CI=true \
- RUN_AS_OPENHANDS=true \
- poetry run pytest -n 5 -raRs --reruns 2 --reruns-delay 3 -s ./tests/runtime --ignore=tests/runtime/test_browsergym_envs.py --durations=10
- env:
- DEBUG: "1"
-
- # The two following jobs (named identically) are to check whether all the runtime tests have passed as the
# "All Runtime Tests Passed" is a required job for PRs to merge
- # Due to this bug: https://github.com/actions/runner/issues/2566, we want to create a job that runs when the
- # prerequisites have been cancelled or failed so merging is disallowed, otherwise Github considers "skipped" as "success"
+ # We can remove this once the config changes
runtime_tests_check_success:
name: All Runtime Tests Passed
- if: ${{ !cancelled() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }}
runs-on: blacksmith-4vcpu-ubuntu-2204
- needs: [test_runtime_root, test_runtime_oh]
steps:
- name: All tests passed
run: echo "All runtime tests have passed successfully!"
- runtime_tests_check_fail:
- name: All Runtime Tests Passed
- if: ${{ cancelled() || contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}
- runs-on: blacksmith-4vcpu-ubuntu-2204
- needs: [test_runtime_root, test_runtime_oh]
- steps:
- - name: Some tests failed
- run: |
- echo "Some runtime tests failed or were cancelled"
- exit 1
update_pr_description:
name: Update PR Description
if: github.event_name == 'pull_request' && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]'