Skip to content

The open-source app everyone uses to manage agents at work

ci: allow dependency changes in cloud eval verification (#13286)

Devin Foley37d7dfb0e3bd
diff --git a/.github/workflows/release-verify.yml b/.github/workflows/release-verify.yml
index 38faeff353..442d198f20 100644
--- a/.github/workflows/release-verify.yml
+++ b/.github/workflows/release-verify.yml
@@ -256,7 +256,7 @@ jobs:
cache: pnpm
- name: Install dependencies
- run: pnpm install --frozen-lockfile
+ run: pnpm install --no-frozen-lockfile
- name: Run deterministic Runner workflow scorer tests
run: pnpm test:runner-workflow-evals
diff --git a/.github/workflows/runner-chaos-evals.yml b/.github/workflows/runner-chaos-evals.yml
index 65a453ee8c..91438bec3a 100644
--- a/.github/workflows/runner-chaos-evals.yml
+++ b/.github/workflows/runner-chaos-evals.yml
@@ -43,7 +43,7 @@ jobs:
cache: pnpm
- name: Install dependencies
- run: pnpm install --frozen-lockfile
+ run: pnpm install --no-frozen-lockfile
- name: Build eval and Runner contracts
run: |
diff --git a/doc/cloud-build-readiness.md b/doc/cloud-build-readiness.md
index e6197ed595..9bb0bee1d4 100644
--- a/doc/cloud-build-readiness.md
+++ b/doc/cloud-build-readiness.md
@@ -49,6 +49,13 @@ job still runs one test worker. The partition covers every suite exactly once;
normal PR and local test groups keep their existing shape. More jobs increase
concurrent runner demand, so compare queue time as well as test duration.
+All release verification installs, including the Runner scorer and chaos evals,
+allow pnpm to refresh an outdated lockfile. Contributor PRs leave lockfile updates
+to the separate refresh bot, so a dependency-changing master commit can arrive
+before that bot's PR merges. Verification must install and test that commit
+without waiting for another merge. The generated lockfile stays in the job's
+workspace; these checks do not commit it back to the repository.
+
The artifact wait runs for up to 30 minutes and reports what is missing. Only
an HTTP 404 means publication is pending; authorization errors, upstream outages,
and identity mismatches fail the job. A failed, cancelled, or skipped prerequisite
diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs
index 08e88a2e40..702edcab8f 100644
--- a/scripts/__tests__/release-verify-workflow.test.mjs
+++ b/scripts/__tests__/release-verify-workflow.test.mjs
@@ -233,7 +233,7 @@ test("release verify workflow covers the same split test surface as stable PR ve
);
assert.match(
verifyWorkflow,
- /runner_workflow_evals:[\s\S]*?Install dependencies\n\s+run: pnpm install --frozen-lockfile[\s\S]*?Run deterministic Runner workflow scorer tests/,
+ /runner_workflow_evals:[\s\S]*?Install dependencies\n\s+run: pnpm install --no-frozen-lockfile[\s\S]*?Run deterministic Runner workflow scorer tests/,
);
assert.match(verifyWorkflow, /pnpm test:runner-workflow-evals/);