Skip to content

The open-source app everyone uses to manage agents at work

ci(runner): prepare target lockfile once for paid validation (#12774)

Dotta6e50ca9d0ae5
diff --git a/.github/workflows/runner-full-stack-e2e.yml b/.github/workflows/runner-full-stack-e2e.yml
index 19bf0fda71..376bc16658 100644
--- a/.github/workflows/runner-full-stack-e2e.yml
+++ b/.github/workflows/runner-full-stack-e2e.yml
@@ -121,7 +121,7 @@ jobs:
AWS_PAID_RUNNER_ENABLED: ${{ vars.RUNNER_E2E_AWS_ENABLED }}
run: |
set -euo pipefail
- github_runner='ubuntu-latest-m'
+ github_runner='ubuntu-latest'
aws_runner='runs-on/fleet=paperclip-public-pr-x64/env=public-ci'
if [ "$AWS_PAID_RUNNER_ENABLED" = true ]; then
@@ -140,9 +140,56 @@ jobs:
echo '::notice title=Paid runner routing::RUNNER_E2E_AWS_ENABLED is not true; using the existing paid runner'
fi
+ target_lock:
+ name: Resolve target pnpm lockfile
+ needs: authorize
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ permissions:
+ contents: read
+ outputs:
+ artifact_id: ${{ steps.upload.outputs.artifact-id }}
+ lock_sha256: ${{ steps.lock.outputs.sha256 }}
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+ with:
+ ref: ${{ needs.authorize.outputs.target_sha }}
+ persist-credentials: false
+
+ - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
+ with:
+ version: 9.15.4
+
+ - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
+ with:
+ node-version: 24
+
+ - name: Resolve target lockfile without lifecycle scripts
+ id: lock
+ run: |
+ set -euo pipefail
+ pnpm install --ignore-scripts --no-frozen-lockfile --lockfile-only
+ test -s pnpm-lock.yaml
+ unexpected="$(git status --short | awk '$2 != "pnpm-lock.yaml" { print }')"
+ if [ -n "$unexpected" ]; then
+ echo "Lockfile resolution changed files other than pnpm-lock.yaml:" >&2
+ echo "$unexpected" >&2
+ exit 1
+ fi
+ echo "sha256=$(sha256sum pnpm-lock.yaml | cut -d ' ' -f 1)" >> "$GITHUB_OUTPUT"
+
+ - name: Upload resolved target lockfile
+ id: upload
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+ with:
+ name: runner-e2e-target-pnpm-lock-${{ github.run_id }}-${{ github.run_attempt }}
+ path: pnpm-lock.yaml
+ retention-days: 30
+ if-no-files-found: error
+
catalog:
name: Validate catalog and select cells
- needs: authorize
+ needs: [authorize, target_lock]
if: github.event_name != 'schedule' || vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true'
runs-on: ubuntu-latest
timeout-minutes: 10
@@ -160,6 +207,26 @@ jobs:
ref: ${{ needs.authorize.outputs.target_sha }}
persist-credentials: false
+ - name: Download resolved target lockfile
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
+ with:
+ artifact-ids: ${{ needs.target_lock.outputs.artifact_id }}
+ path: ${{ runner.temp }}/runner-e2e-target-lock
+
+ - name: Restore resolved target lockfile
+ env:
+ TARGET_SHA: ${{ needs.authorize.outputs.target_sha }}
+ EXPECTED_LOCK_SHA256: ${{ needs.target_lock.outputs.lock_sha256 }}
+ run: |
+ set -euo pipefail
+ test "$(git rev-parse HEAD)" = "$TARGET_SHA"
+ lock="$RUNNER_TEMP/runner-e2e-target-lock/pnpm-lock.yaml"
+ test -f "$lock"
+ test "$(find "$(dirname "$lock")" -type f | wc -l | tr -d ' ')" = 1
+ test "$(sha256sum "$lock" | cut -d ' ' -f 1)" = "$EXPECTED_LOCK_SHA256"
+ cp "$lock" pnpm-lock.yaml
+ test "$(sha256sum pnpm-lock.yaml | cut -d ' ' -f 1)" = "$EXPECTED_LOCK_SHA256"
+
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
with:
version: 9.15.4
@@ -249,7 +316,7 @@ jobs:
daytona_image:
name: Publish verified Daytona image
- needs: [authorize, catalog]
+ needs: [authorize, target_lock, catalog]
runs-on: ubuntu-latest
timeout-minutes: 45
permissions:
@@ -266,6 +333,26 @@ jobs:
ref: ${{ needs.authorize.outputs.target_sha }}
persist-credentials: false
+ - name: Download resolved target lockfile
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
+ with:
+ artifact-ids: ${{ needs.target_lock.outputs.artifact_id }}
+ path: ${{ runner.temp }}/runner-e2e-target-lock
+
+ - name: Restore resolved target lockfile
+ env:
+ TARGET_SHA: ${{ needs.authorize.outputs.target_sha }}
+ EXPECTED_LOCK_SHA256: ${{ needs.target_lock.outputs.lock_sha256 }}
+ run: |
+ set -euo pipefail
+ test "$(git rev-parse HEAD)" = "$TARGET_SHA"
+ lock="$RUNNER_TEMP/runner-e2e-target-lock/pnpm-lock.yaml"
+ test -f "$lock"
+ test "$(find "$(dirname "$lock")" -type f | wc -l | tr -d ' ')" = 1
+ test "$(sha256sum "$lock" | cut -d ' ' -f 1)" = "$EXPECTED_LOCK_SHA256"
+ cp "$lock" pnpm-lock.yaml
+ test "$(sha256sum pnpm-lock.yaml | cut -d ' ' -f 1)" = "$EXPECTED_LOCK_SHA256"
+
- name: No Daytona image needed
id: local_only
if: needs.catalog.outputs.needs_daytona != 'true'
@@ -355,7 +442,7 @@ jobs:
test:
name: ${{ matrix.executionId }}
- needs: [authorize, catalog, daytona_image]
+ needs: [authorize, target_lock, catalog, daytona_image]
# The authorize job selects only one of two literal, reviewed runner labels;
# no dispatch input or repository variable can inject an arbitrary label.
runs-on: ${{ needs.authorize.outputs.test_runner }}
@@ -390,6 +477,26 @@ jobs:
ref: ${{ needs.authorize.outputs.target_sha }}
persist-credentials: false
+ - name: Download resolved target lockfile
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
+ with:
+ artifact-ids: ${{ needs.target_lock.outputs.artifact_id }}
+ path: ${{ runner.temp }}/runner-e2e-target-lock
+
+ - name: Restore resolved target lockfile
+ env:
+ TARGET_SHA: ${{ needs.authorize.outputs.target_sha }}
+ EXPECTED_LOCK_SHA256: ${{ needs.target_lock.outputs.lock_sha256 }}
+ run: |
+ set -euo pipefail
+ test "$(git rev-parse HEAD)" = "$TARGET_SHA"
+ lock="$RUNNER_TEMP/runner-e2e-target-lock/pnpm-lock.yaml"
+ test -f "$lock"
+ test "$(find "$(dirname "$lock")" -type f | wc -l | tr -d ' ')" = 1
+ test "$(sha256sum "$lock" | cut -d ' ' -f 1)" = "$EXPECTED_LOCK_SHA256"
+ cp "$lock" pnpm-lock.yaml
+ test "$(sha256sum pnpm-lock.yaml | cut -d ' ' -f 1)" = "$EXPECTED_LOCK_SHA256"
+
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
with:
version: 9.15.4
@@ -398,7 +505,10 @@ jobs:
with:
node-version: 24
- - run: pnpm install --frozen-lockfile
+ # This job receives provider credentials only in the final paid-test
+ # step. Keep target-selected dependency lifecycle code from running in
+ # the protected environment during setup.
+ - run: pnpm install --frozen-lockfile --ignore-scripts
- name: Build runner TypeScript prerequisites
run: pnpm --filter @paperclipai/paperclip-eval-kernel build
diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md
index 29f8179757..24aa2f8f9a 100644
--- a/tests/runner-e2e/README.md
+++ b/tests/runner-e2e/README.md
@@ -269,12 +269,21 @@ auto-stop/archive/delete values remain as cancellation backstops.
never runs for a pull request or ordinary push. Start the trusted workflow from
the default branch. A CODEOWNER can set the optional `target_branch` input to
any branch in `paperclipai/paperclip`. The authorization job resolves that
-branch to one immutable commit before any checkout. Catalog, image, and paid
-test jobs check out that exact commit. Report sanitization and AWS history
-publication explicitly check out the trusted workflow commit. The workflow
-definition, runner-group permission, and protected-environment deployment still
-come from the default branch. Do not select the target branch in GitHub's **Use
-workflow from** control.
+branch to one immutable commit before any checkout. A separate credential-free
+job checks out the resolved commit and regenerates `pnpm-lock.yaml` once with
+`--ignore-scripts --no-frozen-lockfile --lockfile-only`. It uploads that exact
+lockfile under a run-attempt-scoped artifact ID and records its SHA-256. Catalog,
+image, and paid test jobs download the artifact by ID, verify its digest, and
+restore it before setup or a frozen install. The paid test job disables
+dependency lifecycle scripts, and provider secrets are introduced only in the
+final test step. This permits an authorized target branch to exercise an
+intentionally uncommitted workspace patch while keeping every target job on one
+identical dependency resolution. Report sanitization and AWS history
+publication do not consume the target lockfile; they explicitly check out and
+install from the trusted workflow commit. The workflow definition, runner-group
+permission, and protected-environment deployment still come from the default
+branch. Do not select the target branch in GitHub's **Use workflow from**
+control.
Because this repository is public, manual campaigns fail before checkout unless
the trusted workflow runs from the default branch and both the original actor
@@ -311,8 +320,9 @@ only after the live acceptance ladder in the architecture plan is green.
Set `RUNNER_E2E_AWS_ENABLED=true` to route paid cells to the repository-scoped
ephemeral AWS RunsOn fleet selected by
`runs-on/fleet=paperclip-public-pr-x64/env=public-ci`. Any other value retains
-the existing `ubuntu-latest-m` target. Set `RUNNER_E2E_MAX_PARALLEL` to an
-integer from 1–100 on AWS (default 100); use at least 71 to run the current
+the standard GitHub-hosted `ubuntu-latest` target. Set
+`RUNNER_E2E_MAX_PARALLEL` to an integer from 1–100 on AWS (default 100); use at
+least 71 to run the current
complete catalog in one wave. The fallback runner retains its 1–57 limit and
default of 32. Multi-turn steps are sequential inside their cell while
independent cells overlap. Artifacts and merged HTML/JUnit/normalized reports
diff --git a/tests/runner-e2e/SECURITY.md b/tests/runner-e2e/SECURITY.md
index d99294444c..d730ff4770 100644
--- a/tests/runner-e2e/SECURITY.md
+++ b/tests/runner-e2e/SECURITY.md
@@ -20,10 +20,18 @@ gh api users/LOGIN --jq '{login,id}'
The paid workflows reject manual dispatches when the workflow definition does
not come from the default branch. A trusted dispatcher may name any branch in
`paperclipai/paperclip` as the code under test. The authorization job resolves
-that branch through the GitHub API and passes only its immutable commit SHA to
-the catalog, image, and paid test checkouts. Report sanitization and AWS history
-publication explicitly use the trusted workflow commit. Never run the workflow
-definition from the target branch.
+that branch through the GitHub API and passes only its immutable commit SHA to a
+credential-free target-lock job. That job checks out the commit, regenerates
+`pnpm-lock.yaml` once with lifecycle scripts disabled and lockfile-only mode,
+then uploads the file under a run-attempt-scoped artifact ID. Catalog, image,
+and paid test jobs download that exact artifact by ID, verify its recorded
+SHA-256, and restore it before setup or a frozen dependency install. The lock
+resolver receives no provider credentials and must never run repository
+lifecycle scripts. The paid test job also installs with lifecycle scripts
+disabled, and provider secrets are scoped only to its final test step rather
+than dependency setup. Report sanitization and AWS history publication
+explicitly use the trusted workflow commit and do not consume the target
+lockfile. Never run the workflow definition from the target branch.
The workflows verify both the original actor and triggering actor for every
scheduled or manual attempt, including human reruns. Every
@@ -56,8 +64,8 @@ only `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `OPENROUTER_API_KEY`, and
organization-level Actions secrets: environment scoping is the boundary that
prevents branch or pull-request jobs from requesting them. Require approval
from an account in `RUNNER_E2E_ALLOWED_ACTOR_IDS` for this environment and
-disable administrator bypass. The authorize,
-catalog, image, report, history, and Pages jobs receive none of these secrets.
+disable administrator bypass. The authorize, target-lock, catalog, image,
+report, history, and Pages jobs receive none of these secrets.
Each full-stack matrix cell receives only its selected profile credential, plus
Daytona only for Daytona cells. Secret-bearing and OIDC jobs use frozen installs
without a shared dependency cache.
@@ -75,9 +83,10 @@ the actor gate, environment branch restriction, and protected default branch.
When `RUNNER_E2E_AWS_ENABLED=true`, paid matrix cells use the exact RunsOn fleet
selector `runs-on/fleet=paperclip-public-pr-x64/env=public-ci`, matching the AWS
fleet selected by `pr-trusted.yml` only after its stable numeric-ID trust gate.
-Any other or missing toggle value falls back to the existing `ubuntu-latest-m`
-paid runner and its lower concurrency ceiling. The workflow chooses between
-those two reviewed literal labels; it never evaluates a configured runner label.
+Any other or missing toggle value falls back to the standard GitHub-hosted
+`ubuntu-latest` runner and its lower concurrency ceiling. The workflow chooses
+between those two reviewed literal labels; it never evaluates a configured
+runner label.
Keep both runner targets restricted to `paperclipai/paperclip` and workflows
that independently authorize trusted source revisions. Never let a fork or
diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.ts
index eb0745fb12..d9f23b786c 100644
--- a/tests/runner-e2e/workflow-security.test.ts
+++ b/tests/runner-e2e/workflow-security.test.ts
@@ -66,12 +66,16 @@ describe("public repository paid workflow security", () => {
);
const authorizeJob = fullStack.slice(
fullStack.indexOf(" authorize:"),
+ fullStack.indexOf(" target_lock:"),
+ );
+ const targetLockJob = fullStack.slice(
+ fullStack.indexOf(" target_lock:"),
fullStack.indexOf(" catalog:"),
);
expect(authorizeJob).toContain(
"aws_runner='runs-on/fleet=paperclip-public-pr-x64/env=public-ci'",
);
- expect(authorizeJob).toContain("github_runner='ubuntu-latest-m'");
+ expect(authorizeJob).toContain("github_runner='ubuntu-latest'");
expect(authorizeJob).toContain(
"AWS_PAID_RUNNER_ENABLED: ${{ vars.RUNNER_E2E_AWS_ENABLED }}",
);
@@ -82,13 +86,46 @@ describe("public repository paid workflow security", () => {
"repos/$REPOSITORY/branches/$encoded_branch",
);
expect(authorizeJob).toContain('echo "sha=$target_sha"');
+ expect(authorizeJob).not.toContain("actions/checkout@");
+ expect(authorizeJob).not.toContain("pnpm install");
+ expect(targetLockJob).toContain("name: Resolve target pnpm lockfile");
+ expect(targetLockJob).toContain("needs: authorize");
+ expect(targetLockJob).toContain(
+ "ref: ${{ needs.authorize.outputs.target_sha }}",
+ );
+ expect(targetLockJob).toContain("persist-credentials: false");
+ expect(targetLockJob).toContain(
+ "pnpm install --ignore-scripts --no-frozen-lockfile --lockfile-only",
+ );
+ expect(targetLockJob).toContain(
+ "artifact_id: ${{ steps.upload.outputs.artifact-id }}",
+ );
+ expect(targetLockJob).toContain("lock_sha256:");
+ expect(targetLockJob).toContain(
+ "runner-e2e-target-pnpm-lock-${{ github.run_id }}-${{ github.run_attempt }}",
+ );
+ expect(targetLockJob).not.toContain("name: runner-e2e-paid");
+ expect(targetLockJob).not.toMatch(
+ /(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA)_API_KEY/,
+ );
expect(paidJob).toContain(
"runs-on: ${{ needs.authorize.outputs.test_runner }}",
);
- expect(paidJob).toContain("needs: [authorize, catalog, daytona_image]");
+ expect(paidJob).toContain(
+ "needs: [authorize, target_lock, catalog, daytona_image]",
+ );
expect(paidJob).toContain("name: runner-e2e-paid");
expect(paidJob).toMatch(
- /Reauthorize paid execution before provider access[\s\S]*actions\/checkout@[0-9a-f]{40}[\s\S]*persist-credentials: false/,
+ /Reauthorize paid execution before provider access[\s\S]*actions\/checkout@[0-9a-f]{40}[\s\S]*persist-credentials: false[\s\S]*Download resolved target lockfile/,
+ );
+ const paidInstall = paidJob.indexOf(
+ "pnpm install --frozen-lockfile --ignore-scripts",
+ );
+ const paidExecution = paidJob.indexOf("- name: Run paid cell");
+ expect(paidInstall).toBeGreaterThan(0);
+ expect(paidExecution).toBeGreaterThan(paidInstall);
+ expect(paidJob.slice(0, paidExecution)).not.toMatch(
+ /secrets\.(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA)_API_KEY/,
);
expect(authorizeJob).toContain('echo "max_parallel_limit=100"');
expect(fullStack).toContain('[ "$MAX_PARALLEL_LIMIT" -gt 100 ]');
@@ -101,13 +138,51 @@ describe("public repository paid workflow security", () => {
expect(fullStack).toContain(
"cancel-in-progress: ${{ github.event_name == 'workflow_dispatch' && inputs.target_branch != '' && inputs.target_branch != github.event.repository.default_branch }}",
);
- expect(
- fullStack.match(
- /ref: \$\{\{ needs\.authorize\.outputs\.target_sha \}\}/g,
+ const targetCodeJobs = [
+ fullStack.slice(
+ fullStack.indexOf(" catalog:"),
+ fullStack.indexOf(" daytona_image:"),
+ ),
+ fullStack.slice(
+ fullStack.indexOf(" daytona_image:"),
+ fullStack.indexOf(" test:"),
),
- ).toHaveLength(3);
+ paidJob,
+ ];
+ for (const targetCodeJob of targetCodeJobs) {
+ const checkout = targetCodeJob.indexOf("actions/checkout@");
+ const downloadLock = targetCodeJob.indexOf(
+ "Download resolved target lockfile",
+ );
+ const restoreLock = targetCodeJob.indexOf(
+ "Restore resolved target lockfile",
+ );
+ const setupNode = targetCodeJob.indexOf("actions/setup-node@");
+ const install = targetCodeJob.indexOf("pnpm install --frozen-lockfile");
+ expect(checkout).toBeGreaterThan(0);
+ expect(downloadLock).toBeGreaterThan(checkout);
+ expect(restoreLock).toBeGreaterThan(downloadLock);
+ if (setupNode >= 0) {
+ expect(setupNode).toBeGreaterThan(restoreLock);
+ }
+ if (install >= 0) {
+ expect(install).toBeGreaterThan(restoreLock);
+ }
+ expect(targetCodeJob).toContain(
+ "artifact-ids: ${{ needs.target_lock.outputs.artifact_id }}",
+ );
+ expect(targetCodeJob).toContain(
+ "EXPECTED_LOCK_SHA256: ${{ needs.target_lock.outputs.lock_sha256 }}",
+ );
+ }
+ expect(fullStack.match(/Download resolved target lockfile/g)).toHaveLength(
+ 3,
+ );
+ expect(fullStack.match(/Restore resolved target lockfile/g)).toHaveLength(
+ 3,
+ );
expect(fullStack.match(/ref: \$\{\{ github\.sha \}\}/g)).toHaveLength(2);
- expect(fullStack.match(/persist-credentials: false/g)).toHaveLength(5);
+ expect(fullStack.match(/persist-credentials: false/g)).toHaveLength(6);
expect(fullStack).not.toContain("ref: ${{ inputs.target_branch }}");
expect(fullStack).toContain(
"PAPERCLIP_RUNNER_SOURCE_REVISION=${TARGET_SHA}",
@@ -121,10 +196,12 @@ describe("public repository paid workflow security", () => {
expect(reportJob).not.toContain(
"ref: ${{ needs.authorize.outputs.target_sha }}",
);
+ expect(reportJob).not.toContain("Download resolved target lockfile");
expect(historyJob).toContain("ref: ${{ github.sha }}");
expect(historyJob).not.toContain(
"ref: ${{ needs.authorize.outputs.target_sha }}",
);
+ expect(historyJob).not.toContain("Download resolved target lockfile");
for (const [secret, condition] of Object.entries({
OPENAI_API_KEY: "matrix.credentialName == 'OPENAI_API_KEY'",
ANTHROPIC_API_KEY: "matrix.credentialName == 'ANTHROPIC_API_KEY'",