The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
ci(docker): build each architecture on a native runner instead of QEMU (#12821)
Tonio5488a79eb5c7
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.ymlindex 175863390b..19e9f37935 100644--- a/.github/workflows/docker.yml+++ b/.github/workflows/docker.yml@@ -14,9 +14,10 @@ on:# way.workflow_dispatch:-permissions:- contents: read- packages: write+# Least privilege: nothing at the workflow level; each job declares exactly+# the token scopes it uses (checkout needs contents:read, GHCR pushes need+# packages:write).+permissions: {}# Serialise builds per ref without killing an in-flight one: a newer push# supersedes only the pending slot, so the image build that is already@@ -32,16 +33,51 @@ concurrency:cancel-in-progress: falsejobs:+ # Multi-arch by native runner, not QEMU.+ #+ # This was one job building linux/amd64,linux/arm64 together on an x86+ # runner. The arm64 half is emulated there, and it did not merely run slow:+ # it wedged, every time, in `RUN pnpm --filter @paperclipai/server build`,+ # emitting nothing for 38-45 minutes until `timeout-minutes: 60` killed the+ # job. Verified across three consecutive runs on 2026-09-04; the amd64 half+ # reached `production 5/5` minutes earlier in every one.+ #+ # A timed-out job is reported as *cancelled*, not failed, so the run read+ # "cancelled" and the production image simply stopped publishing without+ # anything going red in an obvious way.+ #+ # It also starved the queue. A run that burns the full hour holds the+ # top-level concurrency slot for that hour, and `cancel-in-progress: false`+ # keeps exactly one pending slot — so with merges arriving faster than one+ # an hour, most runs were superseded before they ever started a job. Five of+ # ten master commits sampled that day never produced an image at all.+ #+ # Each platform now builds on a runner of its own architecture and pushes by+ # digest; `merge` assembles the manifest list. arm64 is kept rather than+ # dropped (the cloud variant below dropped it and is amd64-only) because+ # this is the self-hosted image, and ARM hosts consume it.build-and-push:- runs-on: ubuntu-latest- # The multi-arch (amd64 + QEMU-emulated arm64) production build has- # outgrown 60 minutes: the last runs to finish under the old cap took- # 39-54, and once the build crossed it every job died at the timeout.- # Each hour-long doomed run also held the per-ref concurrency slot, so- # queued master pushes superseded each other and the workflow published- # nothing at all. 120 restores headroom; the cloud job below is- # amd64-only (~10-15 minutes) and keeps its tighter cap.- timeout-minutes: 120+ strategy:+ # Independent legs: one architecture failing should still publish+ # nothing, but it must not also hide the other's logs behind a+ # cancellation.+ fail-fast: false+ matrix:+ include:+ - platform: linux/amd64+ runner: ubuntu-latest+ arch: amd64+ - platform: linux/arm64+ runner: ubuntu-24.04-arm+ arch: arm64+ runs-on: ${{ matrix.runner }}+ # Native builds land well inside this; it is a backstop, not a budget.+ # (The interim fix while this PR landed raised the single QEMU job's cap+ # to 120 minutes; native per-arch legs make that headroom unnecessary.)+ timeout-minutes: 60+ permissions:+ contents: read+ packages: writesteps:- name: Checkoutuses: actions/checkout@v7@@ -184,8 +220,8 @@ jobs:labels: |io.github.paperclipai.schema.last-migration=${{ steps.schema.outputs.last }}io.github.paperclipai.schema.migration-count=${{ steps.schema.outputs.count }}-- - name: Build and push+ - name: Build and push by digest+ id: builduses: docker/build-push-action@v7with:context: .@@ -197,26 +233,124 @@ jobs:PAPERCLIP_BUILD_VERSION=${{ steps.build-version.outputs.version }}PAPERCLIP_BUILD_COMMIT=${{ github.sha }}CLI_TOOLS_CACHE_EPOCH=${{ steps.tools-epoch.outputs.epoch }}- platforms: linux/amd64,linux/arm64- push: true+ platforms: ${{ matrix.platform }}+ # By digest, not by tag: two runners cannot each push the same tag+ # and end up with a manifest list. Each leg publishes an untagged+ # image and `merge` names them together.+ outputs: type=image,name=ghcr.io/${{ github.repository }},push-by-digest=true,name-canonical=true,push=true# Registry-backed BuildKit cache instead of type=gha: the Actions# cache is capped at 10GB per repo, and two multi-arch mode=max jobs# evict each other, so most builds ran effectively cold. The cache# ref lives in ghcr next to the image and is written only by this# workflow (docker.yml runs on master/tag pushes, never on PRs).- cache-from: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache- cache-to: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache,mode=max- tags: ${{ steps.meta.outputs.tags }}+ #+ # Per-arch refs now the legs are separate runners: a shared ref would+ # have each leg overwrite the other's cache on every build.+ cache-from: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-${{ matrix.arch }}+ cache-to: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-${{ matrix.arch }},mode=maxlabels: ${{ steps.meta.outputs.labels }}+ # The digest is the only thing `merge` needs from this job. Carried as an+ # empty file named for it, which is the upstream pattern — the name is+ # the payload, so several legs can upload without colliding on content.+ - name: Export digest+ run: |+ set -euo pipefail+ mkdir -p /tmp/digests+ digest="${{ steps.build.outputs.digest }}"+ test -n "$digest"+ touch "/tmp/digests/${digest#sha256:}"++ - name: Upload digest+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7+ with:+ name: digests-production-${{ matrix.arch }}+ path: /tmp/digests/*+ if-no-files-found: error+ retention-days: 1++ # Names the per-architecture digests as one manifest list under the real+ # tags. Nothing is publicly tagged until this runs, so a half-published+ # multi-arch image is not a state anything can pull.+ merge-and-push:+ needs: build-and-push+ runs-on: ubuntu-latest+ timeout-minutes: 20+ permissions:+ contents: read+ packages: write+ steps:+ # Checked out for `packages/db` (schema labels) and the orphan-reaping+ # script the verification step pipes in.+ - name: Checkout+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7+ with:+ fetch-depth: 0++ - name: Download digests+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8+ with:+ path: /tmp/digests+ pattern: digests-production-*+ merge-multiple: true++ - name: Login to GitHub Container Registry+ uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4+ with:+ registry: ghcr.io+ username: ${{ github.repository_owner }}+ password: ${{ secrets.GITHUB_TOKEN }}++ - name: Set up Docker Buildx+ uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4++ # Repeated from the build job rather than passed between them: job+ # outputs would have to survive a matrix, and this is two `ls` calls.+ - name: Compute schema migration labels+ id: schema+ run: |+ set -euo pipefail+ last=$(ls packages/db/src/migrations/*.sql | sed 's|.*/||' | LC_ALL=C sort | tail -1)+ count=$(ls packages/db/src/migrations/*.sql | wc -l | tr -d ' ')+ echo "last=${last}" >> "$GITHUB_OUTPUT"+ echo "count=${count}" >> "$GITHUB_OUTPUT"++ # Same lane mapping as the build job; this is where it is actually+ # applied, since the legs push untagged. `:canary` is deliberately+ # absent, exactly as in the build job's mapping: the channel tag is+ # moved only by the dist-tag-checked promote_canary_channel job below,+ # so parallel canary-tag builds can never race the channel backwards.+ - name: Docker meta+ id: meta+ uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6+ with:+ images: ghcr.io/${{ github.repository }}+ tags: |+ type=raw,value=nightly,enable=${{ startsWith(github.ref, 'refs/tags/nightly/v') }}+ type=raw,value=beta,enable=${{ startsWith(github.ref, 'refs/tags/beta/v') }}+ type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }}+ type=semver,pattern={{version}},enable=${{ startsWith(github.ref, 'refs/tags/v') }}+ type=semver,pattern={{major}}.{{minor}},enable=${{ startsWith(github.ref, 'refs/tags/v') }}+ type=sha+ labels: |+ io.github.paperclipai.schema.last-migration=${{ steps.schema.outputs.last }}+ io.github.paperclipai.schema.migration-count=${{ steps.schema.outputs.count }}++ - name: Create manifest list and push+ working-directory: /tmp/digests+ run: |+ set -euo pipefail+ docker buildx imagetools create \+ $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \+ $(printf 'ghcr.io/${{ github.repository }}@sha256:%s ' *)+# PID 1 must be an init that reaps adopted orphans. With node there, the# orphans agent runs leave behind are never wait()ed and pin as zombies# until the cgroup pid limit is exhausted and every fork() in the- # container fails. Run against the pushed image rather than a local- # build: the step above is multi-arch with `push: true`, so nothing is- # loaded into the runner's daemon. The cloud variant is FROM production- # and inherits the same ENTRYPOINT, so checking this image covers both.-+ # container fails. Run against the pushed manifest rather than a local+ # build: the legs push by digest, so nothing is loaded into this+ # runner's daemon. The cloud variant is FROM production and inherits the+ # same ENTRYPOINT, so checking this image covers both.- name: Verify PID 1 reaps orphaned processesenv:# Through the environment, not interpolated into the script body, so@@ -239,6 +373,9 @@ jobs:build-and-push-cloud:runs-on: ubuntu-latesttimeout-minutes: 60+ permissions:+ contents: read+ packages: writesteps:- name: Checkoutuses: actions/checkout@v7@@ -459,7 +596,10 @@ jobs:# converges the Docker channel onto the npm channel.promote_canary_channel:if: startsWith(github.ref, 'refs/tags/canary/v')- needs: [build-and-push, build-and-push-cloud]+ # merge-and-push, not build-and-push: the per-arch legs push untagged+ # digests, and the production `sha-*` tags this promotion retags only+ # exist once the manifest merge has named them.+ needs: [merge-and-push, build-and-push-cloud]runs-on: ubuntu-latesttimeout-minutes: 10permissions: