Skip to content

The open-source app everyone uses to manage agents at work

fix(ci): reuse one available Cloud registry cache (#13334)

Devin Foley44f6312cd847
diff --git a/.github/workflows/docker-cloud.yml b/.github/workflows/docker-cloud.yml
index 62025d3a85..81e608c927 100644
--- a/.github/workflows/docker-cloud.yml
+++ b/.github/workflows/docker-cloud.yml
@@ -65,24 +65,6 @@ jobs:
id: tools-epoch
run: echo "epoch=$(date -u +%G-W%V)" >> "$GITHUB_OUTPUT"
- # Each SHA exports its own cache. Import recent first-parent caches so
- # a late older build cannot overwrite a newer build's cache manifest.
- # The legacy ref keeps the first builds warm during the transition.
- - name: Select cloud cache ancestry
- id: cloud-cache
- env:
- CACHE_IMAGE: ghcr.io/${{ github.repository }}
- run: |
- set -euo pipefail
- {
- echo 'sources<<CACHE_SOURCES'
- for commit in $(git rev-list --first-parent --max-count=10 HEAD); do
- echo "type=registry,ref=$CACHE_IMAGE:buildcache-cloud-$commit"
- done
- echo "type=registry,ref=$CACHE_IMAGE:buildcache-cloud"
- echo 'CACHE_SOURCES'
- } >> "$GITHUB_OUTPUT"
-
- name: Setup pnpm
uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
with:
@@ -161,6 +143,14 @@ jobs:
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4
+ # Mixing several historical manifests missed otherwise reusable native
+ # layers on fresh builders. Import the nearest available complete cache.
+ - name: Select cloud cache ancestry
+ id: cloud-cache
+ env:
+ CACHE_IMAGE: ghcr.io/${{ github.repository }}
+ run: node scripts/select-cloud-cache.mjs
+
# Deployment tooling reads these labels from the registry to verify an
# image's schema expectations against a migrator before deploying it,
# without pulling the image. The server refuses to start when the
@@ -225,7 +215,7 @@ jobs:
# Same-SHA builds serialize above; different SHAs never share a
# writable cache ref. Registry layers are content-addressed and
# shared even when cache manifests have separate tags.
- cache-from: ${{ steps.cloud-cache.outputs.sources }}
+ cache-from: ${{ steps.cloud-cache.outputs.source }}
cache-to: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-cloud-${{ github.sha }},mode=max
tags: ${{ steps.meta-cloud.outputs.tags }}
labels: ${{ steps.meta-cloud.outputs.labels }}
diff --git a/.github/workflows/docker-runner-check.yml b/.github/workflows/docker-runner-check.yml
index 7c3b7820b5..d595522292 100644
--- a/.github/workflows/docker-runner-check.yml
+++ b/.github/workflows/docker-runner-check.yml
@@ -32,7 +32,7 @@ jobs:
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4
# Compile the real target, then change source in a disposable context.
- # Require dependency-layer reuse and changed metadata from the real binary.
+ # A fresh builder must import dependencies and produce changed binary metadata.
# No registry credentials, external cache, or image publication.
- name: Verify native build and dependency cache reuse
run: bash scripts/check-docker-runner-cache.sh
diff --git a/doc/DOCKER.md b/doc/DOCKER.md
index 2905bfe957..04f23424b4 100644
--- a/doc/DOCKER.md
+++ b/doc/DOCKER.md
@@ -357,6 +357,14 @@ own native build; no cross-architecture binary is reused. No additional GitHub
Actions cache is created. A cold build also installs the recipe generator and
compiles dependencies, so the savings apply after those layers are available.
+Cloud builds import one registry cache: the first available full-SHA cache in
+the current commit's ten-entry first-parent ancestry, with the legacy cache
+as a final fallback. Each build still exports its own SHA cache with
+`mode=max`. In fresh-builder checks, importing several historical manifests
+missed native layers that a single matching manifest reused. The selector
+inspects metadata after Docker login, stops at the first available cache, and
+permits a cold build if no cache can be read.
+
The application build inherits that stage and still runs the normal server
build, including Cargo, binary staging, and generated-contract checks. Rust
input file times are normalized in both stages so fresh checkouts do not force
@@ -367,12 +375,14 @@ directory as before. Cache misses only cost compilation time.
Pull requests that change the Dockerfile, Docker ignore rules, or Runner native
inputs also build the isolated `runner-build` target in `Docker Runner check`.
The check runs `bash scripts/check-docker-runner-cache.sh` against a disposable
-copy of tracked source and the actual Docker ignore rules. It compiles a baseline,
-changes a Rust metadata constant, and rebuilds. It requires a cached dependency
-build, an unchanged dependency recipe, and changed metadata from the real binary.
-It also verifies that a dependency declaration change alters the recipe. The
-probe exports only small metadata files, avoiding a large image import into the
-Docker daemon. It catches missing embedded inputs before the post-merge build.
-It uses a GitHub-hosted runner with read-only repository access and does not
-publish images or cache artifacts. Allow up to 20 minutes for its cold build and
+copy of tracked source and the actual Docker ignore rules. It compiles a baseline
+and exports a local cache, removes that builder, changes a Rust metadata constant,
+and rebuilds on a fresh builder using only the exported cache. It requires a
+cached dependency build, an unchanged dependency recipe, and changed metadata
+from the real binary. It also verifies that a dependency declaration change
+alters the recipe. The probe exports small metadata results instead of importing
+a large test image into the Docker daemon. Temporary builders and cache files
+are removed afterward. It catches missing embedded inputs before the post-merge
+build. It uses a GitHub-hosted runner with read-only repository access and never
+publishes images or registry caches. Allow up to 20 minutes for its cold build and
source rebuild.
diff --git a/package.json b/package.json
index 40fc577d2c..013c1a44f4 100644
--- a/package.json
+++ b/package.json
@@ -59,7 +59,7 @@
"smoke:posthog-live": "node scripts/smoke/posthog-live.mjs",
"smoke:pipelines-tutorial": "./scripts/smoke/pipelines-tutorial-smoke.sh",
"smoke:terminal-bench-loop-skill": "node scripts/smoke/terminal-bench-loop-skill-smoke.mjs",
- "test:release-registry": "node --test scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs",
+ "test:release-registry": "node --test scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs scripts/select-cloud-cache.test.mjs",
"storybook-visual:baseline": "node scripts/storybook-visual-baseline.mjs",
"test:storybook-visual": "node scripts/storybook-visual-baseline.mjs download && node scripts/storybook-visual-baseline.mjs verify && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts",
"test:storybook-visual:update": "node scripts/storybook-visual-baseline.mjs download && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts --update-snapshots && node scripts/storybook-visual-baseline.mjs pack",
diff --git a/scripts/check-docker-runner-cache.sh b/scripts/check-docker-runner-cache.sh
index 0c8caf81d5..3456948793 100644
--- a/scripts/check-docker-runner-cache.sh
+++ b/scripts/check-docker-runner-cache.sh
@@ -1,10 +1,18 @@
#!/usr/bin/env bash
-# Build the real Docker target twice in a disposable copy of tracked source.
+# Build the real Docker target on two fresh builders using an exported cache.
# Export only metadata, avoiding a multi-gigabyte test image in the daemon.
set -euo pipefail
repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
probe_dir="$(mktemp -d "${TMPDIR:-/tmp}/paperclip-runner-cache.XXXXXX")"
-trap 'rm -rf "$probe_dir"' EXIT
+baseline_builder="${probe_dir##*/}-baseline"
+rebuild_builder="${probe_dir##*/}-rebuild"
+cleanup() {
+ for builder in "$baseline_builder" "$rebuild_builder"; do
+ docker buildx rm "$builder" >/dev/null 2>&1 || true
+ done
+ rm -rf "$probe_dir"
+}
+trap cleanup EXIT
mkdir "$probe_dir/context"
cd "$repo_root"
git ls-files -z | tar -cf - --null -T - | tar -xf - -C "$probe_dir/context"
@@ -21,9 +29,16 @@ FROM scratch AS recipe-proof-export
COPY --from=runner-plan /tmp/runner-recipe.json /recipe.json
DOCKER
build_proof() {
- docker buildx build --file "$probe_dir/cache-probe.Dockerfile" --target cache-proof-export --output "type=local,dest=$probe_dir/$1" --progress plain . 2>&1 | tee "$probe_dir/$1.log"
+ local result="$1" builder="$2"
+ shift 2
+ docker buildx build --builder "$builder" --file "$probe_dir/cache-probe.Dockerfile" --target cache-proof-export --output "type=local,dest=$probe_dir/$result" --progress plain "$@" . 2>&1 | tee "$probe_dir/$result.log"
}
-build_proof baseline
+docker buildx create --name "$baseline_builder" --driver docker-container
+build_proof baseline "$baseline_builder" --cache-to "type=local,dest=$probe_dir/cache,mode=max"
+# Removing the first builder proves the second build cannot use daemon-local
+# state, and releases its disk space before importing the exported cache.
+docker buildx rm "$baseline_builder"
+docker buildx create --name "$rebuild_builder" --driver docker-container
python3 - <<'CHECK'
from pathlib import Path
p=Path('packages/paperclip-runner/runner/crates/runner-core/src/bin/paperclip-runnerd.rs')
@@ -31,7 +46,7 @@ s=p.read_text(); needle='paperclip-runner/runnerd-build-metadata/v1'
assert s.count(needle)==1
p.write_text(s.replace(needle,needle+'-cache-probe'))
CHECK
-build_proof source-change
+build_proof source-change "$rebuild_builder" --cache-from "type=local,src=$probe_dir/cache"
python3 - <<'CHECK'
import os,json,re
from pathlib import Path
@@ -45,12 +60,12 @@ log=(root/'source-change.log').read_text()
step=re.search(r'#(\d+) \[runner-deps[^\n]+ RUN cargo chef cook',log)[1]
assert f'#{step} CACHED' in log
assert 'Compiling paperclip-runner-core' in log
-print('PASS: unchanged dependency recipe and cached cook layer; real binary changed.')
+print('PASS: fresh builder imported compiled dependencies; real binary changed.')
p=Path('packages/paperclip-runner/runner/Cargo.toml')
s=p.read_text(); assert 'serde_json = "1.0"' in s
p.write_text(s.replace('serde_json = "1.0"','serde_json = ">=1.0.0, <2.0.0"'))
CHECK
-docker buildx build --file "$probe_dir/cache-probe.Dockerfile" --target recipe-proof-export --output "type=local,dest=$probe_dir/manifest-change" --progress plain .
+docker buildx build --builder "$rebuild_builder" --file "$probe_dir/cache-probe.Dockerfile" --target recipe-proof-export --output "type=local,dest=$probe_dir/manifest-change" --progress plain .
python3 - <<'CHECK'
from pathlib import Path
import os
diff --git a/scripts/preview-artifacts.test.mjs b/scripts/preview-artifacts.test.mjs
index 15ca6978f2..ebdd9a6122 100644
--- a/scripts/preview-artifacts.test.mjs
+++ b/scripts/preview-artifacts.test.mjs
@@ -4,6 +4,7 @@ import { planArtifacts } from "./preview-artifacts.mjs";
import { readFileSync, mkdtempSync, writeFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
+import { fileURLToPath } from "node:url";
import { gzipSync } from "node:zlib";
import { execFileSync, spawnSync } from "node:child_process";
import { previewManifest, assertMetadata, validateRequest, versionFor, tarManifest, packageExists, imageExists, publishPreview, publishImage } from "./preview-artifacts.mjs";
@@ -217,8 +218,13 @@ test("cloud builds bake the managed runtime identity and verify it before public
test("cloud cache imports are bounded, follow master ancestry, and retain the legacy fallback", () => {
const workflow = readFileSync(new URL("../.github/workflows/docker-cloud.yml", import.meta.url), "utf8");
- const step = workflow.split(" - name: Select cloud cache ancestry")[1].split(" - name: Setup pnpm")[0];
- const script = step.split(" run: |\n")[1].split("\n").map((line) => line.replace(/^ {10}/, "")).join("\n");
+ const selector = workflow.indexOf(" - name: Select cloud cache ancestry");
+ assert.ok(selector > workflow.indexOf(" - name: Login to GitHub Container Registry"));
+ assert.ok(selector > workflow.indexOf(" - name: Set up Docker Buildx"));
+ assert.ok(selector < workflow.indexOf(" - name: Build and push (cloud)"));
+ assert.match(workflow, /run: node scripts\/select-cloud-cache.mjs/);
+ assert.match(workflow, /cache-from: \$\{\{ steps.cloud-cache.outputs.source \}\}/);
+ const script = fileURLToPath(new URL("./select-cloud-cache.mjs", import.meta.url));
const dir = mkdtempSync(path.join(tmpdir(), "cloud-cache-test-"));
const output = path.join(dir, "output");
const env = { ...process.env, GIT_AUTHOR_NAME: "Test", GIT_AUTHOR_EMAIL: "[email protected]", GIT_COMMITTER_NAME: "Test", GIT_COMMITTER_EMAIL: "[email protected]" };
@@ -235,14 +241,25 @@ test("cloud cache imports are bounded, follow master ancestry, and retain the le
git("checkout", "master");
git("merge", "--no-ff", "topic", "-m", "merge topic");
commits.unshift(git("rev-parse", "HEAD"));
- const result = spawnSync("bash", ["-c", script], { cwd: dir, encoding: "utf8", env: { ...env, CACHE_IMAGE: "ghcr.io/paperclipai/paperclip", GITHUB_OUTPUT: output } });
+ const available = `ghcr.io/paperclipai/paperclip:buildcache-cloud-${commits[2]}`;
+ const inspections = path.join(dir, "inspections");
+ writeFileSync(path.join(dir, "docker"), `#!/usr/bin/env node
+const fs = require("node:fs");
+fs.appendFileSync(process.env.CACHE_INSPECTIONS, process.argv.at(-1) + "\\n");
+if (process.argv.at(-1) !== process.env.AVAILABLE_CACHE) {
+ process.stderr.write("manifest unknown");
+ process.exit(1);
+}
+`, { mode: 0o755 });
+ const result = spawnSync(process.execPath, [script], {
+ cwd: dir, encoding: "utf8", env: {
+ ...env, PATH: `${dir}${path.delimiter}${env.PATH}`, CACHE_IMAGE: "ghcr.io/paperclipai/paperclip",
+ GITHUB_OUTPUT: output, AVAILABLE_CACHE: available, CACHE_INSPECTIONS: inspections,
+ },
+ });
assert.equal(result.status, 0, result.stderr);
- assert.deepEqual(readFileSync(output, "utf8").trim().split("\n"), [
- "sources<<CACHE_SOURCES",
- ...commits.slice(0, 10).map((commit) => `type=registry,ref=ghcr.io/paperclipai/paperclip:buildcache-cloud-${commit}`),
- "type=registry,ref=ghcr.io/paperclipai/paperclip:buildcache-cloud",
- "CACHE_SOURCES",
- ]);
+ assert.equal(readFileSync(output, "utf8"), `source=type=registry,ref=${available}\n`);
+ assert.deepEqual(readFileSync(inspections, "utf8").trim().split("\n"), commits.slice(0, 3).map((commit) => `ghcr.io/paperclipai/paperclip:buildcache-cloud-${commit}`));
} finally { rmSync(dir, { recursive: true, force: true }); }
});
diff --git a/scripts/select-cloud-cache.mjs b/scripts/select-cloud-cache.mjs
new file mode 100644
index 0000000000..74b05a533d
--- /dev/null
+++ b/scripts/select-cloud-cache.mjs
@@ -0,0 +1,66 @@
+#!/usr/bin/env node
+import { execFileSync } from "node:child_process";
+import { appendFileSync } from "node:fs";
+import { pathToFileURL } from "node:url";
+
+export function cloudCacheCandidates(image, commits) {
+ if (!/^ghcr\.io\/[a-z0-9._-]+\/[a-z0-9._-]+$/.test(image ?? "")) {
+ throw new Error("Expected a GHCR owner/repository cache image.");
+ }
+ if (!Array.isArray(commits) || commits.length === 0 || commits.some((sha) => !/^[a-f0-9]{40}$/.test(sha))) {
+ throw new Error("Cloud cache ancestry requires full commit SHAs.");
+ }
+ return [
+ ...[...new Set(commits)].slice(0, 10).map((sha) => `${image}:buildcache-cloud-${sha}`),
+ `${image}:buildcache-cloud`,
+ ];
+}
+
+export async function selectCloudCache(image, commits, {
+ exists = registryCacheExists,
+ log = console.log,
+} = {}) {
+ for (const ref of cloudCacheCandidates(image, commits)) {
+ try {
+ if (!await exists(ref)) continue;
+ log(`Using cloud cache: ${ref}`);
+ return `type=registry,ref=${ref}`;
+ } catch {
+ // Cache availability must not turn an otherwise valid build into a
+ // failure. A later ancestor may still be available during a rollout.
+ log(`Could not inspect cloud cache ${ref}; trying the next ancestor.`);
+ }
+ }
+ log("No cloud cache is available; this build will populate one.");
+ return "";
+}
+
+function registryCacheExists(ref) {
+ try {
+ // Use the preceding Docker login, including for private registry caches.
+ // Inspect metadata only: no layer download and no image execution.
+ execFileSync("docker", ["buildx", "imagetools", "inspect", "--raw", ref], {
+ timeout: 10_000,
+ maxBuffer: 1024 * 1024,
+ stdio: ["ignore", "pipe", "pipe"],
+ });
+ return true;
+ } catch (error) {
+ if (/manifest unknown|not found|NAME_UNKNOWN/i.test(String(error.stderr ?? ""))) return false;
+ throw error;
+ }
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+ try {
+ if (!process.env.GITHUB_OUTPUT) throw new Error("GITHUB_OUTPUT is required.");
+ const commits = execFileSync("git", ["rev-list", "--first-parent", "--max-count=10", "HEAD"], {
+ encoding: "utf8",
+ }).trim().split("\n");
+ const source = await selectCloudCache(process.env.CACHE_IMAGE, commits, { exists: registryCacheExists });
+ appendFileSync(process.env.GITHUB_OUTPUT, `source=${source}\n`);
+ } catch (error) {
+ console.error(error.message);
+ process.exitCode = 1;
+ }
+}
diff --git a/scripts/select-cloud-cache.test.mjs b/scripts/select-cloud-cache.test.mjs
new file mode 100644
index 0000000000..d160fa4f04
--- /dev/null
+++ b/scripts/select-cloud-cache.test.mjs
@@ -0,0 +1,65 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { cloudCacheCandidates, selectCloudCache } from "./select-cloud-cache.mjs";
+
+const image = "ghcr.io/paperclipai/paperclip";
+const commits = ["a".repeat(40), "b".repeat(40), "c".repeat(40)];
+const candidates = cloudCacheCandidates(image, commits);
+
+test("a same-SHA rerun imports only its existing cache", async () => {
+ const inspected = [];
+ const source = await selectCloudCache(image, commits, {
+ exists: async (ref) => { inspected.push(ref); return true; }, log() {},
+ });
+ assert.equal(source, `type=registry,ref=${candidates[0]}`);
+ assert.deepEqual(inspected, candidates.slice(0, 1));
+});
+
+test("a new merge imports only its nearest available ancestor", async () => {
+ const inspected = [];
+ const source = await selectCloudCache(image, commits, {
+ exists: async (ref) => { inspected.push(ref); return ref === candidates[1]; }, log() {},
+ });
+ assert.equal(source, `type=registry,ref=${candidates[1]}`);
+ assert.deepEqual(inspected, candidates.slice(0, 2));
+ assert.equal(source.includes("\n"), false);
+});
+
+test("a still-building parent falls back to an older completed cache", async () => {
+ assert.equal(await selectCloudCache(image, commits, {
+ exists: async (ref) => ref === candidates[2], log() {},
+ }), `type=registry,ref=${candidates[2]}`);
+});
+
+test("the legacy cache is used only if no SHA cache exists", async () => {
+ const inspected = [];
+ assert.equal(await selectCloudCache(image, commits, {
+ exists: async (ref) => { inspected.push(ref); return ref === candidates.at(-1); }, log() {},
+ }), `type=registry,ref=${candidates.at(-1)}`);
+ assert.deepEqual(inspected, candidates);
+});
+
+test("missing caches permit a cold build", async () => {
+ assert.equal(await selectCloudCache(image, commits, { exists: async () => false, log() {} }), "");
+});
+
+test("a failed lookup can fall back without failing image publication", async () => {
+ const messages = [];
+ assert.equal(await selectCloudCache(image, commits, {
+ exists: async (ref) => {
+ if (ref === candidates[0]) throw new Error("registry temporarily unavailable");
+ return true;
+ },
+ log: (message) => messages.push(message),
+ }), `type=registry,ref=${candidates[1]}`);
+ assert.match(messages[0], /Could not inspect cloud cache/);
+});
+
+test("ancestry is bounded, deduplicated, and rejects output injection", () => {
+ const many = Array.from({ length: 20 }, (_, i) => i.toString(16).padStart(40, "0"));
+ assert.equal(cloudCacheCandidates(image, many).length, 11);
+ assert.deepEqual(cloudCacheCandidates(image, [commits[0], commits[0]]), [candidates[0], candidates.at(-1)]);
+ assert.throws(() => cloudCacheCandidates(`${image}\nsource=untrusted`, commits));
+ assert.throws(() => cloudCacheCandidates(image, ["master"]));
+ assert.throws(() => cloudCacheCandidates(image, []));
+});