π OpenHands: AI-Driven Development
Optimize dockerfile by consolidating and reordering steps (#9549)
Ray Myers5cc47ee592b4
diff --git a/.github/workflows/ghcr-build.yml b/.github/workflows/ghcr-build.ymlindex 078feac160..bfe4db3e99 100644--- a/.github/workflows/ghcr-build.yml+++ b/.github/workflows/ghcr-build.yml@@ -40,7 +40,8 @@ jobs:# Only build nikolaik on PRs, otherwise build both nikolaik and ubuntu.if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; thenjson=$(jq -n -c '[- { image: "nikolaik/python-nodejs:python3.12-nodejs22", tag: "nikolaik" }+ { image: "nikolaik/python-nodejs:python3.12-nodejs22", tag: "nikolaik" },+ { image: "ubuntu:24.04", tag: "ubuntu" }]')elsejson=$(jq -n -c '[@@ -58,9 +59,6 @@ jobs:permissions:contents: readpackages: write- outputs:- # Since this job uses outputs it cannot use matrix- hash_from_app_image: ${{ steps.get_hash_in_app_image.outputs.hash_from_app_image }}steps:- name: Checkoutuses: actions/checkout@v4@@ -86,24 +84,11 @@ jobs:if: "!github.event.pull_request.head.repo.fork"run: |./containers/build.sh -i openhands -o ${{ env.REPO_OWNER }} --push- - name: Build app image- if: "github.event.pull_request.head.repo.fork"- run: |- ./containers/build.sh -i openhands -o ${{ env.REPO_OWNER }} --load- - name: Get hash in App Image- id: get_hash_in_app_image- run: |- # Run the build script in the app image- docker run -e SANDBOX_USER_ID=0 -v /var/run/docker.sock:/var/run/docker.sock ghcr.io/${{ env.REPO_OWNER }}/openhands:${{ env.RELEVANT_SHA }} /bin/bash -c "mkdir -p containers/runtime; python3 openhands/runtime/utils/runtime_build.py --base_image ${{ env.BASE_IMAGE_FOR_HASH_EQUIVALENCE_TEST }} --build_folder containers/runtime --force_rebuild" 2>&1 | tee docker-outputs.txt- # Get the hash from the build script- hash_from_app_image=$(cat docker-outputs.txt | grep "Hash for docker build directory" | awk -F "): " '{print $2}' | uniq | head -n1)- echo "hash_from_app_image=$hash_from_app_image" >> $GITHUB_OUTPUT- echo "Hash from app image: $hash_from_app_image"# Builds the runtime Docker imagesghcr_build_runtime:name: Build Image- runs-on: blacksmith-4vcpu-ubuntu-2204+ runs-on: blacksmith-8vcpu-ubuntu-2204if: "!(github.event_name == 'push' && startsWith(github.ref, 'refs/tags/ext-v'))"permissions:contents: read@@ -130,22 +115,13 @@ jobs:- name: Set up Docker Buildxid: buildxuses: docker/setup-buildx-action@v3+ - name: Install poetry via pipx+ run: pipx install poetry- name: Set up Pythonuses: useblacksmith/setup-python@v6with:python-version: '3.12'- - name: Cache Poetry dependencies- uses: useblacksmith/cache@v5- with:- path: |- ~/.cache/pypoetry- ~/.virtualenvs- key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}- # This is the one that saves the cache, the others set 'lookup-only: true'- restore-keys: |- ${{ runner.os }}-poetry-- - name: Install poetry via pipx- run: pipx install poetry+ cache: poetry- name: Install Python dependencies using Poetryrun: make install-python-dependencies POETRY_GROUP=main INSTALL_PLAYWRIGHT=0- name: Create source distribution and Dockerfile@@ -190,61 +166,6 @@ jobs:name: runtime-src-${{ matrix.base_image.tag }}path: containers/runtime- verify_hash_equivalence_in_runtime_and_app:- name: Verify Hash Equivalence in Runtime and Docker images- runs-on: blacksmith-4vcpu-ubuntu-2204- needs: [ghcr_build_runtime, ghcr_build_app]- strategy:- fail-fast: false- matrix:- base_image: ['nikolaik']- env:- BASE_IMAGE_FOR_HASH_EQUIVALENCE_TEST: nikolaik/python-nodejs:python3.12-nodejs22- steps:- - uses: actions/checkout@v4- with:- ref: ${{ github.event.pull_request.head.sha }}- - name: Cache Poetry dependencies- uses: useblacksmith/cache@v5- with:- path: |- ~/.cache/pypoetry- ~/.virtualenvs- key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}- lookup-only: true- restore-keys: |- ${{ runner.os }}-poetry-- - name: Set up Python- uses: useblacksmith/setup-python@v6- with:- python-version: '3.12'- - name: Install poetry via pipx- run: pipx install poetry- - name: Install Python dependencies using Poetry- run: make install-python-dependencies POETRY_GROUP=main INSTALL_PLAYWRIGHT=0- - name: Get hash in App Image- run: |- echo "Hash from app image: ${{ needs.ghcr_build_app.outputs.hash_from_app_image }}"- echo "hash_from_app_image=${{ needs.ghcr_build_app.outputs.hash_from_app_image }}" >> $GITHUB_ENV-- - name: Get hash using code (development mode)- run: |- mkdir -p containers/runtime- poetry run python3 openhands/runtime/utils/runtime_build.py --base_image ${{ env.BASE_IMAGE_FOR_HASH_EQUIVALENCE_TEST }} --build_folder containers/runtime --force_rebuild > output.txt 2>&1- hash_from_code=$(cat output.txt | grep "Hash for docker build directory" | awk -F "): " '{print $2}' | uniq | head -n1)- echo "hash_from_code=$hash_from_code" >> $GITHUB_ENV-- - name: Compare hashes- run: |- echo "Hash from App Image: ${{ env.hash_from_app_image }}"- echo "Hash from Code: ${{ env.hash_from_code }}"- if [ "${{ env.hash_from_app_image }}" = "${{ env.hash_from_code }}" ]; then- echo "Hashes match!"- else- echo "Hashes do not match!"- exit 1- fi-# Run unit tests with the Docker runtime Docker images as roottest_runtime_root:name: RT Unit Tests (Root)@@ -276,25 +197,17 @@ jobs:load: truetags: ghcr.io/${{ env.REPO_OWNER }}/runtime:${{ env.RELEVANT_SHA }}-${{ matrix.base_image.tag }}context: containers/runtime- - name: Cache Poetry dependencies- uses: useblacksmith/cache@v5- with:- path: |- ~/.cache/pypoetry- ~/.virtualenvs- key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}- lookup-only: true- restore-keys: |- ${{ runner.os }}-poetry-+ - name: Install poetry via pipx+ run: pipx install poetry- name: Set up Pythonuses: useblacksmith/setup-python@v6with:python-version: '3.12'- - name: Install poetry via pipx- run: pipx install poetry+ cache: poetry- name: Install Python dependencies using Poetryrun: make install-python-dependencies INSTALL_PLAYWRIGHT=0- name: Run docker runtime tests+ shell: bashrun: |# We install pytest-xdist in order to run tests across CPUspoetry run pip install pytest-xdist@@ -346,25 +259,17 @@ jobs:load: truetags: ghcr.io/${{ env.REPO_OWNER }}/runtime:${{ env.RELEVANT_SHA }}-${{ matrix.base_image.tag }}context: containers/runtime- - name: Cache Poetry dependencies- uses: useblacksmith/cache@v5- with:- path: |- ~/.cache/pypoetry- ~/.virtualenvs- key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}- lookup-only: true- restore-keys: |- ${{ runner.os }}-poetry-+ - name: Install poetry via pipx+ run: pipx install poetry- name: Set up Pythonuses: useblacksmith/setup-python@v6with:python-version: '3.12'- - name: Install poetry via pipx- run: pipx install poetry+ cache: poetry- name: Install Python dependencies using Poetryrun: make install-python-dependencies POETRY_GROUP=main,test,runtime INSTALL_PLAYWRIGHT=0- name: Run runtime tests+ shell: bashrun: |# We install pytest-xdist in order to run tests across CPUspoetry run pip install pytest-xdist@@ -391,7 +296,7 @@ jobs:name: All Runtime Tests Passedif: ${{ !cancelled() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }}runs-on: blacksmith-4vcpu-ubuntu-2204- needs: [test_runtime_root, test_runtime_oh, verify_hash_equivalence_in_runtime_and_app]+ needs: [test_runtime_root, test_runtime_oh]steps:- name: All tests passedrun: echo "All runtime tests have passed successfully!"@@ -400,7 +305,7 @@ jobs:name: All Runtime Tests Passedif: ${{ cancelled() || contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}runs-on: blacksmith-4vcpu-ubuntu-2204- needs: [test_runtime_root, test_runtime_oh, verify_hash_equivalence_in_runtime_and_app]+ needs: [test_runtime_root, test_runtime_oh]steps:- name: Some tests failedrun: |@@ -425,6 +330,7 @@ jobs:PR_NUMBER: ${{ github.event.pull_request.number }}REPO: ${{ github.repository }}SHORT_SHA: ${{ steps.short_sha.outputs.SHORT_SHA }}+ shell: bashrun: |echo "updating PR description"DOCKER_RUN_COMMAND="docker run -it --rm \diff --git a/openhands/runtime/utils/runtime_templates/Dockerfile.j2 b/openhands/runtime/utils/runtime_templates/Dockerfile.j2index f9561574b9..433dd315b1 100644--- a/openhands/runtime/utils/runtime_templates/Dockerfile.j2+++ b/openhands/runtime/utils/runtime_templates/Dockerfile.j2@@ -13,23 +13,33 @@ ENV POETRY_VIRTUALENVS_PATH=/openhands/poetry \{% macro setup_base_system() %}# Install base system dependencies++{% if 'ubuntu' in base_image %}RUN apt-get update && \apt-get install -y --no-install-recommends \wget curl ca-certificates sudo apt-utils git jq tmux build-essential ripgrep ffmpeg \- {%- if 'ubuntu' in base_image and (base_image.endswith(':latest') or base_image.endswith(':24.04')) -%}+ {%- if (base_image.endswith(':latest') or base_image.endswith(':24.04')) -%}libgl1 \{%- else %}libgl1-mesa-glx \{% endif -%}libasound2-plugins libatomic1 && \- {%- if 'ubuntu' in base_image -%}curl -fsSL https://deb.nodesource.com/setup_22.x | bash - && \TZ=Etc/UTC DEBIAN_FRONTEND=noninteractive \apt-get install -y --no-install-recommends nodejs python3.12 python-is-python3 python3-pip python3.12-venv && \- corepack enable yarn && \- {% endif -%}apt-get clean && \rm -rf /var/lib/apt/lists/*+{% endif %}++{% if 'ubuntu' not in base_image %}+RUN apt-get update && \+ apt-get install -y --no-install-recommends \+ wget curl ca-certificates sudo apt-utils git jq tmux build-essential ripgrep ffmpeg \+ libgl1-mesa-glx \+ libasound2-plugins libatomic1 && \+ apt-get clean && \+ rm -rf /var/lib/apt/lists/*+{% endif %}{% if 'ubuntu' in base_image %}RUN ln -s "$(dirname $(which node))/corepack" /usr/local/bin/corepack && \@@ -105,28 +115,22 @@ RUN rm -rf ${OPENVSCODE_SERVER_ROOT}/extensions/{handlebars,pug,json,diff,grunt,# Install all dependenciesWORKDIR /openhands/code-# Configure micromamba and poetry-RUN /openhands/micromamba/bin/micromamba config set changeps1 False && \+RUN \+ /openhands/micromamba/bin/micromamba config set changeps1 False && \/openhands/micromamba/bin/micromamba run -n openhands poetry config virtualenvs.path /openhands/poetry && \- /openhands/micromamba/bin/micromamba run -n openhands poetry env use python3.12--# Install project dependencies in smaller chunks-RUN /openhands/micromamba/bin/micromamba run -n openhands poetry install --only main --no-interaction --no-root--RUN /openhands/micromamba/bin/micromamba run -n openhands poetry install --only runtime --no-interaction --no-root--# Install playwright and its dependencies-RUN apt-get update && \- /openhands/micromamba/bin/micromamba run -n openhands poetry run pip install playwright && \- /openhands/micromamba/bin/micromamba run -n openhands poetry run playwright install --with-deps chromium--# Set environment variables and permissions-RUN /openhands/micromamba/bin/micromamba run -n openhands poetry run python -c "import sys; print('OH_INTERPRETER_PATH=' + sys.executable)" >> /etc/environment && \+ /openhands/micromamba/bin/micromamba run -n openhands poetry env use python3.12 && \+ # Install project dependencies+ /openhands/micromamba/bin/micromamba run -n openhands poetry install --only main,runtime --no-interaction --no-root && \+ # Update and install additional tools+ # (There used to be an "apt-get update" here, hopefully we can skip it.)+ /openhands/micromamba/bin/micromamba run -n openhands poetry run playwright install --with-deps chromium && \+ # Set environment variables+ /openhands/micromamba/bin/micromamba run -n openhands poetry run python -c "import sys; print('OH_INTERPRETER_PATH=' + sys.executable)" >> /etc/environment && \+ # Set permissionschmod -R g+rws /openhands/poetry && \- mkdir -p /openhands/workspace && chmod -R g+rws,o+rw /openhands/workspace--# Clear caches-RUN /openhands/micromamba/bin/micromamba run -n openhands poetry cache clear --all . -n && \+ mkdir -p /openhands/workspace && chmod -R g+rws,o+rw /openhands/workspace && \+ # Clean up+ /openhands/micromamba/bin/micromamba run -n openhands poetry cache clear --all . -n && \apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \/openhands/micromamba/bin/micromamba clean --all@@ -166,6 +170,8 @@ COPY ./code/pyproject.toml ./code/poetry.lock /openhands/code/# ================================================================{% endif %}+{{ setup_vscode_server() }}+# ================================================================# Copy Project source files# ================================================================@@ -175,7 +181,7 @@ COPY ./code/pyproject.toml ./code/poetry.lock /openhands/code/COPY ./code/openhands /openhands/code/openhandsRUN chmod a+rwx /openhands/code/openhands/__init__.py-{{ setup_vscode_server() }}+# ================================================================# END: Build from versioned image