Skip to content

The open-source app everyone uses to manage agents at work

ci(docker): build each architecture on a native runner instead of QEMU (#12821)

Tonio5488a79eb5c7
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index 175863390b..19e9f37935 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -14,9 +14,10 @@ on:
# way.
workflow_dispatch:
-permissions:
- contents: read
- packages: write
+# Least privilege: nothing at the workflow level; each job declares exactly
+# the token scopes it uses (checkout needs contents:read, GHCR pushes need
+# packages:write).
+permissions: {}
# Serialise builds per ref without killing an in-flight one: a newer push
# supersedes only the pending slot, so the image build that is already
@@ -32,16 +33,51 @@ concurrency:
cancel-in-progress: false
jobs:
+ # Multi-arch by native runner, not QEMU.
+ #
+ # This was one job building linux/amd64,linux/arm64 together on an x86
+ # runner. The arm64 half is emulated there, and it did not merely run slow:
+ # it wedged, every time, in `RUN pnpm --filter @paperclipai/server build`,
+ # emitting nothing for 38-45 minutes until `timeout-minutes: 60` killed the
+ # job. Verified across three consecutive runs on 2026-09-04; the amd64 half
+ # reached `production 5/5` minutes earlier in every one.
+ #
+ # A timed-out job is reported as *cancelled*, not failed, so the run read
+ # "cancelled" and the production image simply stopped publishing without
+ # anything going red in an obvious way.
+ #
+ # It also starved the queue. A run that burns the full hour holds the
+ # top-level concurrency slot for that hour, and `cancel-in-progress: false`
+ # keeps exactly one pending slot — so with merges arriving faster than one
+ # an hour, most runs were superseded before they ever started a job. Five of
+ # ten master commits sampled that day never produced an image at all.
+ #
+ # Each platform now builds on a runner of its own architecture and pushes by
+ # digest; `merge` assembles the manifest list. arm64 is kept rather than
+ # dropped (the cloud variant below dropped it and is amd64-only) because
+ # this is the self-hosted image, and ARM hosts consume it.
build-and-push:
- runs-on: ubuntu-latest
- # The multi-arch (amd64 + QEMU-emulated arm64) production build has
- # outgrown 60 minutes: the last runs to finish under the old cap took
- # 39-54, and once the build crossed it every job died at the timeout.
- # Each hour-long doomed run also held the per-ref concurrency slot, so
- # queued master pushes superseded each other and the workflow published
- # nothing at all. 120 restores headroom; the cloud job below is
- # amd64-only (~10-15 minutes) and keeps its tighter cap.
- timeout-minutes: 120
+ strategy:
+ # Independent legs: one architecture failing should still publish
+ # nothing, but it must not also hide the other's logs behind a
+ # cancellation.
+ fail-fast: false
+ matrix:
+ include:
+ - platform: linux/amd64
+ runner: ubuntu-latest
+ arch: amd64
+ - platform: linux/arm64
+ runner: ubuntu-24.04-arm
+ arch: arm64
+ runs-on: ${{ matrix.runner }}
+ # Native builds land well inside this; it is a backstop, not a budget.
+ # (The interim fix while this PR landed raised the single QEMU job's cap
+ # to 120 minutes; native per-arch legs make that headroom unnecessary.)
+ timeout-minutes: 60
+ permissions:
+ contents: read
+ packages: write
steps:
- name: Checkout
uses: actions/checkout@v7
@@ -184,8 +220,8 @@ jobs:
labels: |
io.github.paperclipai.schema.last-migration=${{ steps.schema.outputs.last }}
io.github.paperclipai.schema.migration-count=${{ steps.schema.outputs.count }}
-
- - name: Build and push
+ - name: Build and push by digest
+ id: build
uses: docker/build-push-action@v7
with:
context: .
@@ -197,26 +233,124 @@ jobs:
PAPERCLIP_BUILD_VERSION=${{ steps.build-version.outputs.version }}
PAPERCLIP_BUILD_COMMIT=${{ github.sha }}
CLI_TOOLS_CACHE_EPOCH=${{ steps.tools-epoch.outputs.epoch }}
- platforms: linux/amd64,linux/arm64
- push: true
+ platforms: ${{ matrix.platform }}
+ # By digest, not by tag: two runners cannot each push the same tag
+ # and end up with a manifest list. Each leg publishes an untagged
+ # image and `merge` names them together.
+ outputs: type=image,name=ghcr.io/${{ github.repository }},push-by-digest=true,name-canonical=true,push=true
# Registry-backed BuildKit cache instead of type=gha: the Actions
# cache is capped at 10GB per repo, and two multi-arch mode=max jobs
# evict each other, so most builds ran effectively cold. The cache
# ref lives in ghcr next to the image and is written only by this
# workflow (docker.yml runs on master/tag pushes, never on PRs).
- cache-from: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache
- cache-to: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache,mode=max
- tags: ${{ steps.meta.outputs.tags }}
+ #
+ # Per-arch refs now the legs are separate runners: a shared ref would
+ # have each leg overwrite the other's cache on every build.
+ cache-from: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-${{ matrix.arch }}
+ cache-to: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-${{ matrix.arch }},mode=max
labels: ${{ steps.meta.outputs.labels }}
+ # The digest is the only thing `merge` needs from this job. Carried as an
+ # empty file named for it, which is the upstream pattern — the name is
+ # the payload, so several legs can upload without colliding on content.
+ - name: Export digest
+ run: |
+ set -euo pipefail
+ mkdir -p /tmp/digests
+ digest="${{ steps.build.outputs.digest }}"
+ test -n "$digest"
+ touch "/tmp/digests/${digest#sha256:}"
+
+ - name: Upload digest
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+ with:
+ name: digests-production-${{ matrix.arch }}
+ path: /tmp/digests/*
+ if-no-files-found: error
+ retention-days: 1
+
+ # Names the per-architecture digests as one manifest list under the real
+ # tags. Nothing is publicly tagged until this runs, so a half-published
+ # multi-arch image is not a state anything can pull.
+ merge-and-push:
+ needs: build-and-push
+ runs-on: ubuntu-latest
+ timeout-minutes: 20
+ permissions:
+ contents: read
+ packages: write
+ steps:
+ # Checked out for `packages/db` (schema labels) and the orphan-reaping
+ # script the verification step pipes in.
+ - name: Checkout
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+ with:
+ fetch-depth: 0
+
+ - name: Download digests
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
+ with:
+ path: /tmp/digests
+ pattern: digests-production-*
+ merge-multiple: true
+
+ - name: Login to GitHub Container Registry
+ uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
+ with:
+ registry: ghcr.io
+ username: ${{ github.repository_owner }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4
+
+ # Repeated from the build job rather than passed between them: job
+ # outputs would have to survive a matrix, and this is two `ls` calls.
+ - name: Compute schema migration labels
+ id: schema
+ run: |
+ set -euo pipefail
+ last=$(ls packages/db/src/migrations/*.sql | sed 's|.*/||' | LC_ALL=C sort | tail -1)
+ count=$(ls packages/db/src/migrations/*.sql | wc -l | tr -d ' ')
+ echo "last=${last}" >> "$GITHUB_OUTPUT"
+ echo "count=${count}" >> "$GITHUB_OUTPUT"
+
+ # Same lane mapping as the build job; this is where it is actually
+ # applied, since the legs push untagged. `:canary` is deliberately
+ # absent, exactly as in the build job's mapping: the channel tag is
+ # moved only by the dist-tag-checked promote_canary_channel job below,
+ # so parallel canary-tag builds can never race the channel backwards.
+ - name: Docker meta
+ id: meta
+ uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6
+ with:
+ images: ghcr.io/${{ github.repository }}
+ tags: |
+ type=raw,value=nightly,enable=${{ startsWith(github.ref, 'refs/tags/nightly/v') }}
+ type=raw,value=beta,enable=${{ startsWith(github.ref, 'refs/tags/beta/v') }}
+ type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
+ type=semver,pattern={{version}},enable=${{ startsWith(github.ref, 'refs/tags/v') }}
+ type=semver,pattern={{major}}.{{minor}},enable=${{ startsWith(github.ref, 'refs/tags/v') }}
+ type=sha
+ labels: |
+ io.github.paperclipai.schema.last-migration=${{ steps.schema.outputs.last }}
+ io.github.paperclipai.schema.migration-count=${{ steps.schema.outputs.count }}
+
+ - name: Create manifest list and push
+ working-directory: /tmp/digests
+ run: |
+ set -euo pipefail
+ docker buildx imagetools create \
+ $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+ $(printf 'ghcr.io/${{ github.repository }}@sha256:%s ' *)
+
# PID 1 must be an init that reaps adopted orphans. With node there, the
# orphans agent runs leave behind are never wait()ed and pin as zombies
# until the cgroup pid limit is exhausted and every fork() in the
- # container fails. Run against the pushed image rather than a local
- # build: the step above is multi-arch with `push: true`, so nothing is
- # loaded into the runner's daemon. The cloud variant is FROM production
- # and inherits the same ENTRYPOINT, so checking this image covers both.
-
+ # container fails. Run against the pushed manifest rather than a local
+ # build: the legs push by digest, so nothing is loaded into this
+ # runner's daemon. The cloud variant is FROM production and inherits the
+ # same ENTRYPOINT, so checking this image covers both.
- name: Verify PID 1 reaps orphaned processes
env:
# Through the environment, not interpolated into the script body, so
@@ -239,6 +373,9 @@ jobs:
build-and-push-cloud:
runs-on: ubuntu-latest
timeout-minutes: 60
+ permissions:
+ contents: read
+ packages: write
steps:
- name: Checkout
uses: actions/checkout@v7
@@ -459,7 +596,10 @@ jobs:
# converges the Docker channel onto the npm channel.
promote_canary_channel:
if: startsWith(github.ref, 'refs/tags/canary/v')
- needs: [build-and-push, build-and-push-cloud]
+ # merge-and-push, not build-and-push: the per-arch legs push untagged
+ # digests, and the production `sha-*` tags this promotion retags only
+ # exist once the manifest merge has named them.
+ needs: [merge-and-push, build-and-push-cloud]
runs-on: ubuntu-latest
timeout-minutes: 10
permissions: