π OpenHands: AI-Driven Development
feat(enterprise): Build ARM64 image for enterprise, use native runners for all images (#13921)
Joe Lavertyab64a65f25c9
diff --git a/.github/workflows/ghcr-build.yml b/.github/workflows/ghcr-build.ymlindex c73df9b403..f57c01b694 100644--- a/.github/workflows/ghcr-build.yml+++ b/.github/workflows/ghcr-build.yml@@ -33,45 +33,46 @@ jobs:runs-on: ubuntu-latestoutputs:base_image: ${{ steps.define-base-images.outputs.base_image }}- platforms: ${{ steps.define-base-images.outputs.platforms }}+ architectures: ${{ steps.define-base-images.outputs.architectures }}steps:- name: Define base imagesshell: bashid: define-base-imagesrun: |+ architectures='["amd64", "arm64"]'if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then- platforms="linux/amd64"- json=$(jq -n -c --arg platforms "$platforms" '[- { image: "nikolaik/python-nodejs:python3.12-nodejs22-slim", tag: "nikolaik", platforms: $platforms }+ json=$(jq -n -c '[+ { image: "nikolaik/python-nodejs:python3.12-nodejs22-slim", tag: "nikolaik" }]')else- platforms="linux/amd64,linux/arm64"- json=$(jq -n -c --arg platforms "$platforms" '[- { image: "nikolaik/python-nodejs:python3.12-nodejs22-slim", tag: "nikolaik", platforms: $platforms },- { image: "ubuntu:24.04", tag: "ubuntu", platforms: $platforms }+ json=$(jq -n -c '[+ { image: "nikolaik/python-nodejs:python3.12-nodejs22-slim", tag: "nikolaik" },+ { image: "ubuntu:24.04", tag: "ubuntu" }]')fiecho "base_image=$json" >> "$GITHUB_OUTPUT"- echo "platforms=$platforms" >> "$GITHUB_OUTPUT"+ echo "architectures=$architectures" >> "$GITHUB_OUTPUT"- # Builds the OpenHands Docker images+ # Builds the OpenHands Docker images (one per architecture, natively)ghcr_build_app:- name: Build App Image- runs-on: ubuntu-22.04+ name: Build App Image (${{ matrix.arch }})+ runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-22.04' }}if: "!(github.event_name == 'push' && startsWith(github.ref, 'refs/tags/ext-v'))"needs: define-matrix+ outputs:+ # All arch variants produce the same base tags, so any entry works+ base_tags: ${{ steps.build.outputs.base_tags }}permissions:contents: readpackages: write+ strategy:+ matrix:+ arch: ${{ fromJson(needs.define-matrix.outputs.architectures) }}steps:- name: Checkoutuses: actions/checkout@v6with:ref: ${{ github.event.pull_request.head.sha }}- - name: Set up QEMU- uses: docker/[email protected]- with:- image: tonistiigi/binfmt:latest- name: Login to GHCRuses: docker/login-action@v4with:@@ -85,31 +86,77 @@ jobs:run: |echo REPO_OWNER=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]') >> $GITHUB_ENV- name: Build and push app image+ id: buildif: "!github.event.pull_request.head.repo.fork"run: |- ./containers/build.sh -i openhands -o ${{ env.REPO_OWNER }} --push -p ${{ needs.define-matrix.outputs.platforms }}+ ./containers/build.sh -i openhands -o ${{ env.REPO_OWNER }} --push --arch ${{ matrix.arch }}- # Builds the runtime Docker images- ghcr_build_runtime:- name: Build Runtime Image+ # Output base tags (without arch suffix) for the merge job+ ./containers/build.sh -i openhands -o ${{ env.REPO_OWNER }} --arch ${{ matrix.arch }} --dry+ BASE_TAGS=$(jq -r '.base_tags | join("\n")' docker-build-dry.json)+ echo "base_tags<<EOF" >> "$GITHUB_OUTPUT"+ echo "$BASE_TAGS" >> "$GITHUB_OUTPUT"+ echo "EOF" >> "$GITHUB_OUTPUT"++ # Merges per-architecture app images into a multi-arch manifest+ ghcr_build_app_merge:+ name: Merge App Multi-Arch Manifestruns-on: ubuntu-22.04+ needs: [define-matrix, ghcr_build_app]+ if: github.event.pull_request.head.repo.fork != true+ permissions:+ packages: write+ steps:+ - name: Login to GHCR+ uses: docker/login-action@v3+ with:+ registry: ghcr.io+ username: ${{ github.repository_owner }}+ password: ${{ secrets.GITHUB_TOKEN }}+ - name: Set up Docker Buildx+ uses: docker/setup-buildx-action@v3+ - name: Merge multi-arch manifest+ run: |+ ARCHS='${{ join(fromJson(needs.define-matrix.outputs.architectures), ' ') }}'+ TAGS="${{ needs.ghcr_build_app.outputs.base_tags }}"+ while IFS= read -r tag; do+ [[ -z "$tag" ]] && continue+ sources=""+ for arch in $ARCHS; do+ if ! docker buildx imagetools inspect "${tag}-${arch}" > /dev/null 2>&1; then+ echo "::error::Missing image ${tag}-${arch}"+ exit 1+ fi+ sources+=" ${tag}-${arch}"+ done+ echo "Creating manifest for $tag from:$sources"+ docker buildx imagetools create -t "$tag" $sources+ done <<< "$TAGS"++ # Builds the runtime Docker images (one per architecture x base_image, natively)+ ghcr_build_runtime:+ name: Build Runtime Image (${{ matrix.base_image.tag }}, ${{ matrix.arch }})+ runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-22.04' }}if: "!(github.event_name == 'push' && startsWith(github.ref, 'refs/tags/ext-v'))"permissions:contents: readpackages: writeneeds: define-matrix+ outputs:+ # Keyed by base_image tag so the merge job can access per-variant tags.+ # Matrix outputs from different entries with the same key overwrite each other,+ # but all arch variants of the same base_image produce identical base tags.+ base_tags_nikolaik: ${{ steps.params.outputs.base_tags_nikolaik }}+ base_tags_ubuntu: ${{ steps.params.outputs.base_tags_ubuntu }}strategy:matrix:base_image: ${{ fromJson(needs.define-matrix.outputs.base_image) }}+ arch: ${{ fromJson(needs.define-matrix.outputs.architectures) }}steps:- name: Checkoutuses: actions/checkout@v6with:ref: ${{ github.event.pull_request.head.sha }}- - name: Set up QEMU- uses: docker/[email protected]- with:- image: tonistiigi/binfmt:latest- name: Login to GHCRuses: docker/login-action@v4with:@@ -137,16 +184,22 @@ jobs:run: |echo SHORT_SHA=$(git rev-parse --short "$RELEVANT_SHA") >> $GITHUB_ENV- name: Determine docker build params+ id: paramsif: github.event.pull_request.head.repo.fork != trueshell: bashrun: |-- ./containers/build.sh -i runtime -o ${{ env.REPO_OWNER }} -t ${{ matrix.base_image.tag }} --dry -p ${{ matrix.base_image.platforms }}+ ./containers/build.sh -i runtime -o ${{ env.REPO_OWNER }} -t ${{ matrix.base_image.tag }} --arch ${{ matrix.arch }} --dryDOCKER_BUILD_JSON=$(jq -c . < docker-build-dry.json)echo "DOCKER_TAGS=$(echo "$DOCKER_BUILD_JSON" | jq -r '.tags | join(",")')" >> $GITHUB_ENVecho "DOCKER_PLATFORM=$(echo "$DOCKER_BUILD_JSON" | jq -r '.platform')" >> $GITHUB_ENVecho "DOCKER_BUILD_ARGS=$(echo "$DOCKER_BUILD_JSON" | jq -r '.build_args | join(",")')" >> $GITHUB_ENV++ # Output base tags (without arch suffix) keyed by base_image tag for the merge job+ BASE_TAGS=$(echo "$DOCKER_BUILD_JSON" | jq -r '.base_tags | join("\n")')+ echo "base_tags_${{ matrix.base_image.tag }}<<EOF" >> "$GITHUB_OUTPUT"+ echo "$BASE_TAGS" >> "$GITHUB_OUTPUT"+ echo "EOF" >> "$GITHUB_OUTPUT"- name: Build and push runtime image ${{ matrix.base_image.image }}if: github.event.pull_request.head.repo.fork != trueuses: docker/build-push-action@v6@@ -154,9 +207,8 @@ jobs:push: truetags: ${{ env.DOCKER_TAGS }}platforms: ${{ env.DOCKER_PLATFORM }}- # Caching directives to boost performance- cache-from: type=registry,ref=ghcr.io/${{ env.REPO_OWNER }}/runtime:buildcache-${{ matrix.base_image.tag }}- cache-to: type=registry,ref=ghcr.io/${{ env.REPO_OWNER }}/runtime:buildcache-${{ matrix.base_image.tag }},mode=max+ cache-from: type=registry,ref=ghcr.io/${{ env.REPO_OWNER }}/runtime:buildcache-${{ matrix.base_image.tag }}-${{ matrix.arch }}+ cache-to: type=registry,ref=ghcr.io/${{ env.REPO_OWNER }}/runtime:buildcache-${{ matrix.base_image.tag }}-${{ matrix.arch }},mode=maxbuild-args: ${{ env.DOCKER_BUILD_ARGS }}context: containers/runtimeprovenance: false@@ -171,18 +223,64 @@ jobs:if: github.event.pull_request.head.repo.forkuses: actions/upload-artifact@v7with:- name: runtime-src-${{ matrix.base_image.tag }}+ name: runtime-src-${{ matrix.base_image.tag }}-${{ matrix.arch }}path: containers/runtime- ghcr_build_enterprise:- name: Push Enterprise Image+ # Merges per-architecture runtime images into multi-arch manifests+ ghcr_build_runtime_merge:+ name: Merge Runtime Multi-Arch Manifestruns-on: ubuntu-22.04+ needs: [define-matrix, ghcr_build_runtime]+ if: github.event.pull_request.head.repo.fork != true+ permissions:+ packages: write+ steps:+ - name: Login to GHCR+ uses: docker/login-action@v3+ with:+ registry: ghcr.io+ username: ${{ github.repository_owner }}+ password: ${{ secrets.GITHUB_TOKEN }}+ - name: Set up Docker Buildx+ uses: docker/setup-buildx-action@v3+ - name: Merge multi-arch manifests+ run: |+ ARCHS='${{ join(fromJson(needs.define-matrix.outputs.architectures), ' ') }}'++ # Merge all runtime base_image variants+ for variant_tags in \+ "${{ needs.ghcr_build_runtime.outputs.base_tags_nikolaik }}" \+ "${{ needs.ghcr_build_runtime.outputs.base_tags_ubuntu }}"; do+ while IFS= read -r tag; do+ [[ -z "$tag" ]] && continue+ sources=""+ for arch in $ARCHS; do+ if ! docker buildx imagetools inspect "${tag}-${arch}" > /dev/null 2>&1; then+ echo "::error::Missing image ${tag}-${arch}"+ exit 1+ fi+ sources+=" ${tag}-${arch}"+ done+ echo "Creating manifest for $tag from:$sources"+ docker buildx imagetools create -t "$tag" $sources+ done <<< "$variant_tags"+ done++ ghcr_build_enterprise:+ name: Push Enterprise Image (${{ matrix.arch }})+ runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-22.04' }}permissions:contents: readpackages: write- needs: [define-matrix, ghcr_build_app]+ needs: [define-matrix, ghcr_build_app_merge]# Do not build enterprise in forksif: github.event.pull_request.head.repo.fork != true+ outputs:+ # Tags without arch suffix, for the merge job+ base_tags: ${{ steps.meta_base.outputs.tags }}+ strategy:+ matrix:+ arch: ${{ fromJson(needs.define-matrix.outputs.architectures) }}steps:- name: Checkoutuses: actions/checkout@v6@@ -205,6 +303,28 @@ jobs:- name: Extract metadata (tags, labels) for Dockerid: metauses: docker/metadata-action@v6+ with:+ images: ghcr.io/openhands/enterprise-server+ tags: |+ type=ref,event=branch+ type=ref,event=pr+ type=sha+ type=sha,format=long+ type=semver,pattern={{version}}+ type=semver,pattern={{major}}.{{minor}}+ type=semver,pattern={{major}}+ type=match,pattern=cloud-\d+\.\d+\.\d++ flavor: |+ latest=auto+ prefix=+ suffix=-${{ matrix.arch }}+ env:+ DOCKER_METADATA_PR_HEAD_SHA: true++ # Also compute base tags (no arch suffix) for the merge job output+ - name: Extract base metadata for merge+ id: meta_base+ uses: docker/metadata-action@v5with:images: ghcr.io/openhands/enterprise-servertags: |@@ -222,6 +342,7 @@ jobs:suffix=env:DOCKER_METADATA_PR_HEAD_SHA: true+- name: Determine app image tagshell: bashrun: |@@ -238,12 +359,49 @@ jobs:labels: ${{ steps.meta.outputs.labels }}build-args: |OPENHANDS_VERSION=${{ env.OPENHANDS_DOCKER_TAG }}- platforms: linux/amd64+ platforms: linux/${{ matrix.arch }}# Add build provenanceprovenance: true# Add build attestations for better securitysbom: true+ # Merges per-architecture enterprise images into a multi-arch manifest+ ghcr_build_enterprise_merge:+ name: Merge Enterprise Multi-Arch Manifest+ runs-on: ubuntu-22.04+ permissions:+ packages: write+ needs: [define-matrix, ghcr_build_enterprise]+ if: github.event.pull_request.head.repo.fork != true+ steps:+ - name: Login to GHCR+ uses: docker/login-action@v3+ with:+ registry: ghcr.io+ username: ${{ github.repository_owner }}+ password: ${{ secrets.GITHUB_TOKEN }}++ - name: Set up Docker Buildx+ uses: docker/setup-buildx-action@v3++ - name: Merge multi-arch manifest+ run: |+ ARCHS='${{ join(fromJson(needs.define-matrix.outputs.architectures), ' ') }}'+ TAGS="${{ needs.ghcr_build_enterprise.outputs.base_tags }}"+ while IFS= read -r tag; do+ [[ -z "$tag" ]] && continue+ sources=""+ for arch in $ARCHS; do+ if ! docker buildx imagetools inspect "${tag}-${arch}" > /dev/null 2>&1; then+ echo "::error::Missing image ${tag}-${arch}"+ exit 1+ fi+ sources+=" ${tag}-${arch}"+ done+ echo "Creating manifest for $tag from:$sources"+ docker buildx imagetools create -t "$tag" $sources+ done <<< "$TAGS"+# "All Runtime Tests Passed" is a required job for PRs to merge# We can remove this once the config changesruntime_tests_check_success:@@ -256,7 +414,7 @@ jobs:update_pr_description:name: Update PR Descriptionif: github.event_name == 'pull_request' && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]'- needs: [ghcr_build_runtime]+ needs: [ghcr_build_runtime_merge]runs-on: ubuntu-22.04steps:- name: Checkoutdiff --git a/containers/build.sh b/containers/build.shindex eaf923c20e..79dc59c417 100755--- a/containers/build.sh+++ b/containers/build.sh@@ -8,18 +8,18 @@ push=0load=0tag_suffix=""dry_run=0-platform_override=""+arch_suffix=""# Function to display usage informationusage() {- echo "Usage: $0 -i <image_name> [-o <org_name>] [--push] [--load] [-t <tag_suffix>] [-p <platform>] [--dry]"+ echo "Usage: $0 -i <image_name> [-o <org_name>] [--push] [--load] [-t <tag_suffix>] [--dry] [--arch <arch>]"echo " -i: Image name (required)"echo " -o: Organization name"echo " --push: Push the image"echo " --load: Load the image"echo " -t: Tag suffix"- echo " -p: Platform(s) to build for (e.g. linux/amd64 or linux/amd64,linux/arm64)"echo " --dry: Don't build, only create build-args.json"+ echo " --arch: Architecture suffix (e.g. amd64 or arm64). Appends -<arch> to tags and forces single-platform build"exit 1}@@ -31,8 +31,8 @@ while [[ $# -gt 0 ]]; do--push) push=1; shift ;;--load) load=1; shift ;;-t) tag_suffix="$2"; shift 2 ;;- -p) platform_override="$2"; shift 2 ;;--dry) dry_run=1; shift ;;+ --arch) arch_suffix="$2"; shift 2 ;;*) usage ;;esacdone@@ -78,7 +78,7 @@ if [[ -n $tag_suffix ]]; thendonefi-echo "Tags: ${tags[@]}"+echo "Tags (before arch suffix): ${tags[@]}"if [[ "$image_name" == "openhands" ]]; thendir="./containers/app"@@ -113,10 +113,21 @@ if [[ -n "$DOCKER_IMAGE_TAG" ]]; thentags+=("$DOCKER_IMAGE_TAG")fi+# Apply architecture suffix for split-arch builds (after all tags are collected)+if [[ -n "$arch_suffix" ]]; then+ cache_tag+="-${arch_suffix}"+ for i in "${!tags[@]}"; do+ tags[$i]="${tags[$i]}-${arch_suffix}"+ done+ # Force single-platform build for this architecture+ arch_platform="linux/${arch_suffix}"+fi+DOCKER_REPOSITORY="$DOCKER_REGISTRY/$DOCKER_ORG/$DOCKER_IMAGE"DOCKER_REPOSITORY=${DOCKER_REPOSITORY,,} # lowercaseecho "Repo: $DOCKER_REPOSITORY"echo "Base dir: $DOCKER_BASE_DIR"+echo "Tags: ${tags[@]}"args=""full_tags=()@@ -125,7 +136,6 @@ for tag in "${tags[@]}"; dofull_tags+=("$DOCKER_REPOSITORY:$tag")done-if [[ $push -eq 1 ]]; thenargs+=" --push"args+=" --cache-to=type=registry,ref=$DOCKER_REPOSITORY:$cache_tag,mode=max"@@ -138,8 +148,8 @@ fiecho "Args: $args"# Determine the platform(s) to build for-if [[ -n "$platform_override" ]]; then- platform="$platform_override"+if [[ -n "$arch_platform" ]]; then+ platform="$arch_platform"elif [[ $load -eq 1 ]]; then# When loading, build only for the current platformplatform=$(docker version -f '{{.Server.Os}}/{{.Server.Arch}}')@@ -149,13 +159,24 @@ elsefiif [[ $dry_run -eq 1 ]]; thenecho "Dry Run is enabled. Writing build config to docker-build-dry.json"+ # Compute base tags (arch suffix stripped) for use by merge jobs+ base_tags=()+ for ftag in "${full_tags[@]}"; do+ if [[ -n "$arch_suffix" ]]; then+ base_tags+=("${ftag%-${arch_suffix}}")+ else+ base_tags+=("$ftag")+ fi+ donejq -n \--argjson tags "$(printf '%s\n' "${full_tags[@]}" | jq -R . | jq -s .)" \+ --argjson base_tags "$(printf '%s\n' "${base_tags[@]}" | jq -R . | jq -s .)" \--arg platform "$platform" \--arg openhands_build_version "$OPENHANDS_BUILD_VERSION" \--arg dockerfile "$dir/Dockerfile" \'{tags: $tags,+ base_tags: $base_tags,platform: $platform,build_args: ["OPENHANDS_BUILD_VERSION=" + $openhands_build_version@@ -174,7 +195,7 @@ docker buildx build \$args \--build-arg OPENHANDS_BUILD_VERSION="$OPENHANDS_BUILD_VERSION" \--cache-from=type=registry,ref=$DOCKER_REPOSITORY:$cache_tag \- --cache-from=type=registry,ref=$DOCKER_REPOSITORY:$cache_tag_base-main \+ --cache-from=type=registry,ref=$DOCKER_REPOSITORY:${cache_tag_base}-main${arch_suffix:+-${arch_suffix}} \--platform $platform \--provenance=false \-f "$dir/Dockerfile" \