Skip to content

The open-source app everyone uses to manage agents at work

ci: split runner verification from build (#13142)

Nicky Leach2a05b5ed3457
diff --git a/.github/workflows/pr-trusted.yml b/.github/workflows/pr-trusted.yml
index 40168b4565..0dd8d149a9 100644
--- a/.github/workflows/pr-trusted.yml
+++ b/.github/workflows/pr-trusted.yml
@@ -553,7 +553,7 @@ jobs:
# Preserve the legacy required-check name while the underlying work runs in parallel.
name: verify
if: ${{ always() }}
- needs: [gate, policy, typecheck_release_registry, general_tests, build, docker_context_integrity]
+ needs: [gate, policy, typecheck_release_registry, general_tests, verify_paperclip_runner, build, docker_context_integrity]
runs-on: ${{ needs.gate.outputs.runner }}
timeout-minutes: 5
@@ -564,6 +564,7 @@ jobs:
POLICY_RESULT: ${{ needs.policy.result }}
TYPECHECK_RELEASE_REGISTRY_RESULT: ${{ needs.typecheck_release_registry.result }}
GENERAL_TESTS_RESULT: ${{ needs.general_tests.result }}
+ RUNNER_VERIFICATION_RESULT: ${{ needs.verify_paperclip_runner.result }}
BUILD_RESULT: ${{ needs.build.result }}
DOCKER_CONTEXT_INTEGRITY_RESULT: ${{ needs.docker_context_integrity.result }}
run: |
@@ -572,12 +573,14 @@ jobs:
true)
test "$TYPECHECK_RELEASE_REGISTRY_RESULT" = "success"
test "$GENERAL_TESTS_RESULT" = "success"
+ test "$RUNNER_VERIFICATION_RESULT" = "success"
test "$BUILD_RESULT" = "success"
test "$DOCKER_CONTEXT_INTEGRITY_RESULT" = "success"
;;
false)
test "$TYPECHECK_RELEASE_REGISTRY_RESULT" = "skipped"
test "$GENERAL_TESTS_RESULT" = "skipped"
+ test "$RUNNER_VERIFICATION_RESULT" = "skipped"
test "$BUILD_RESULT" = "skipped"
test "$DOCKER_CONTEXT_INTEGRITY_RESULT" = "skipped"
;;
@@ -587,8 +590,8 @@ jobs:
;;
esac
- build:
- name: Build
+ verify_paperclip_runner:
+ name: Verify Paperclip Runner
needs: [gate, policy]
if: ${{ needs.gate.outputs.full_ci == 'true' }}
runs-on: ${{ needs.gate.outputs.runner }}
@@ -633,6 +636,49 @@ jobs:
- name: Verify Paperclip Runner
run: pnpm --filter @paperclipai/paperclip-runner check:all
+ build:
+ name: Build
+ needs: [gate, policy]
+ if: ${{ needs.gate.outputs.full_ci == 'true' }}
+ runs-on: ${{ needs.gate.outputs.runner }}
+ timeout-minutes: 20
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+ with:
+ persist-credentials: false
+
+ - name: Setup Node.js for pnpm bootstrap
+ uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
+ with:
+ node-version: 24
+
+ - name: Setup pnpm
+ uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
+ env:
+ NPM_CONFIG_AUDIT: "false"
+ NPM_CONFIG_FUND: "false"
+ NPM_CONFIG_UPDATE_NOTIFIER: "false"
+ with:
+ version: 9.15.4
+
+ - name: Restore regenerated PR lockfile (if policy uploaded one)
+ if: needs.policy.outputs.lockfile_regenerated == '1'
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
+ with:
+ name: pr-lockfile
+ path: .
+
+ - name: Setup Node.js
+ uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
+ with:
+ node-version: 24
+ cache: pnpm
+
+ - name: Install dependencies
+ run: pnpm install --frozen-lockfile
+
- name: Build Runner Evalbook viewer
run: pnpm --filter @paperclipai/paperclip-runner build:issue-thread
diff --git a/.github/workflows/release-verify.yml b/.github/workflows/release-verify.yml
index a8bc1abbf7..7193f38c84 100644
--- a/.github/workflows/release-verify.yml
+++ b/.github/workflows/release-verify.yml
@@ -191,8 +191,8 @@ jobs:
- name: Run deterministic Runner workflow scorer tests
run: pnpm test:runner-workflow-evals
- build:
- name: Build
+ verify_paperclip_runner:
+ name: Verify Paperclip Runner
runs-on: ubuntu-latest
timeout-minutes: 20
permissions:
@@ -221,5 +221,32 @@ jobs:
- name: Verify Paperclip Runner
run: pnpm --filter @paperclipai/paperclip-runner check:all
+ build:
+ name: Build
+ runs-on: ubuntu-latest
+ timeout-minutes: 20
+ permissions:
+ contents: read
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+ with:
+ ref: ${{ inputs.ref }}
+ persist-credentials: false
+
+ - name: Setup pnpm
+ uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
+ with:
+ version: 9.15.4
+
+ - name: Setup Node.js
+ uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
+ with:
+ node-version: 24
+
+ - name: Install dependencies
+ run: pnpm install --no-frozen-lockfile
+
- name: Build
run: pnpm build
diff --git a/scripts/__tests__/e2e-shard.test.mjs b/scripts/__tests__/e2e-shard.test.mjs
index 1fed746a27..00893bc8bc 100644
--- a/scripts/__tests__/e2e-shard.test.mjs
+++ b/scripts/__tests__/e2e-shard.test.mjs
@@ -222,6 +222,7 @@ test("the trusted PR workflow limits full CI to merge-relevant stack layers", ()
for (const jobId of [
"typecheck_release_registry",
"general_tests",
+ "verify_paperclip_runner",
"build",
"verify_serialized_server",
"canary_dry_run",
@@ -243,11 +244,16 @@ test("the trusted PR workflow limits full CI to merge-relevant stack layers", ()
const verify = jobs.get("verify");
assert.match(
verify,
- /^ {4}needs: \[gate, policy, typecheck_release_registry, general_tests, build, docker_context_integrity\]$/m,
+ /^ {4}needs: \[gate, policy, typecheck_release_registry, general_tests, verify_paperclip_runner, build, docker_context_integrity\]$/m,
);
assert.match(verify, /POLICY_RESULT: \$\{\{ needs\.policy\.result \}\}/);
assert.match(verify, /test "\$TYPECHECK_RELEASE_REGISTRY_RESULT" = "skipped"/);
assert.match(verify, /test "\$GENERAL_TESTS_RESULT" = "skipped"/);
+ // Runner verification must participate in the legacy aggregate required
+ // check, or a runner regression could be merged while `verify` succeeds.
+ assert.match(verify, /RUNNER_VERIFICATION_RESULT: \$\{\{ needs\.verify_paperclip_runner\.result \}\}/);
+ assert.match(verify, /test "\$RUNNER_VERIFICATION_RESULT" = "success"/);
+ assert.match(verify, /test "\$RUNNER_VERIFICATION_RESULT" = "skipped"/);
assert.match(verify, /test "\$BUILD_RESULT" = "skipped"/);
// Both halves of the docker-context lane's gating: the result must be
// wired into the aggregate's env AND asserted successful on full CI —
@@ -327,7 +333,7 @@ test("the trusted PR workflow regenerates stale stacked lockfiles", () => {
const restoreSteps = workflow.match(
/- name: Restore regenerated PR lockfile \(if policy uploaded one\)\n if: needs\.policy\.outputs\.lockfile_regenerated == '1'/g,
) ?? [];
- assert.equal(restoreSteps.length, 6, "every downstream install job must restore a required regenerated artifact");
+ assert.equal(restoreSteps.length, 7, "every downstream install job must restore a required regenerated artifact");
assert.doesNotMatch(
workflow,
/- name: Restore regenerated PR lockfile \(if policy uploaded one\)[\s\S]{0,220}continue-on-error:/,
diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs
index aaf3179215..8b18a89039 100644
--- a/scripts/__tests__/release-verify-workflow.test.mjs
+++ b/scripts/__tests__/release-verify-workflow.test.mjs
@@ -202,6 +202,10 @@ test("release verify workflow covers the same split test surface as stable PR ve
);
assert.match(verifyWorkflow, /pnpm test:runner-workflow-evals/);
+ const buildJob = verifyWorkflow.match(/ build:\n[\s\S]*?(?=\n [A-Za-z0-9_-]+:|$)/)?.[0] ?? "";
+ assert.match(buildJob, /persist-credentials: false/);
+ assert.doesNotMatch(buildJob, /cache: pnpm/);
+
for (const group of [
"general-server",
"general-workspaces-a",