Skip to content

The open-source app everyone uses to manage agents at work

fix(ci): isolate chaos verification by caller workflow (#13208)

Devin Foleyfc06f7f05f42
diff --git a/.github/workflows/runner-chaos-evals.yml b/.github/workflows/runner-chaos-evals.yml
index 3131959522..42f8d6f307 100644
--- a/.github/workflows/runner-chaos-evals.yml
+++ b/.github/workflows/runner-chaos-evals.yml
@@ -12,7 +12,9 @@ on:
type: string
concurrency:
- group: runner-chaos-evals-${{ inputs.ref || github.ref }}
+ # Reusable calls inherit the caller's workflow name. Cloud readiness and
+ # Release verify the same SHA independently and must not cancel each other.
+ group: runner-chaos-evals-${{ github.workflow }}-${{ inputs.ref || github.ref }}
cancel-in-progress: true
jobs:
diff --git a/doc/cloud-build-readiness.md b/doc/cloud-build-readiness.md
index 68f7827437..6522f03e05 100644
--- a/doc/cloud-build-readiness.md
+++ b/doc/cloud-build-readiness.md
@@ -50,6 +50,10 @@ successfully verified a real master commit.
## Timing and rollout
+The reusable Runner chaos workflow scopes concurrency to the caller workflow
+and source ref. Cloud readiness and the npm release can verify the same commit
+at the same time. They must not cancel each other's required test job.
+
Measure the complete path from a master merge to a healthy target running that
exact commit. Keep readiness and deployment as separate milestones:
diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs
index 3ff0d35b27..9def9e3800 100644
--- a/scripts/__tests__/release-verify-workflow.test.mjs
+++ b/scripts/__tests__/release-verify-workflow.test.mjs
@@ -14,6 +14,27 @@ function readWorkflow(name) {
return readFileSync(path.join(repoRoot, ".github/workflows", name), "utf8");
}
+test("chaos verification isolates callers that verify the same source commit", () => {
+ const chaosWorkflow = readWorkflow("runner-chaos-evals.yml");
+ const group = chaosWorkflow.match(/^ group: (.+)$/m)?.[1];
+ assert.ok(group, "chaos verification must define its concurrency group");
+
+ // GitHub supplies the top-level caller's workflow name to reusable calls.
+ const resolveGroup = (caller, ref) => group
+ .replaceAll("${{ github.workflow }}", readWorkflow(caller).match(/^name: (.+)$/m)[1])
+ .replaceAll("${{ inputs.ref || github.ref }}", ref)
+ .toLowerCase();
+ const sha = "a".repeat(40);
+ const callers = ["cloud-readiness.yml", "release.yml", "runner-chaos-evals.yml"];
+ const groups = callers.map((caller) => resolveGroup(caller, sha));
+ assert.equal(new Set(groups).size, callers.length,
+ "Cloud readiness, Release, and standalone evals must not cancel each other");
+ assert.ok(groups.every((value) => !value.includes("${{")), "resolve every group input");
+ assert.notEqual(resolveGroup("cloud-readiness.yml", sha),
+ resolveGroup("cloud-readiness.yml", "b".repeat(40)), "different sources remain independent");
+ assert.match(chaosWorkflow, /cancel-in-progress: true/);
+});
+
test("release workflow delegates stable and canary verification to the reusable workflow", () => {
const releaseWorkflow = readWorkflow("release.yml");