Skip to content

The open-source app everyone uses to manage agents at work

ci: optimize checks for stacked pull requests (#12507)

Dotta39b8ee296054
diff --git a/.github/workflows/pr-trusted.yml b/.github/workflows/pr-trusted.yml
index a2e34b4cd1..5bc040f992 100644
--- a/.github/workflows/pr-trusted.yml
+++ b/.github/workflows/pr-trusted.yml
@@ -22,6 +22,7 @@ jobs:
timeout-minutes: 5
outputs:
runner: ${{ steps.route.outputs.runner }}
+ full_ci: ${{ steps.scope.outputs.full_ci }}
steps:
- name: Validate PR identity and select runner
@@ -217,6 +218,41 @@ jobs:
echo "runner=$aws_runner" >> "$GITHUB_OUTPUT"
echo '::notice title=AWS CI routing::Using an ephemeral RunsOn Fleet runner'
+ - name: Select stacked PR CI scope
+ id: scope
+ shell: bash
+ env:
+ STACK_JSON: ${{ toJSON(github.event.pull_request.stack) }}
+ PR_BASE_REF: ${{ github.event.pull_request.base.ref }}
+ run: |
+ set -euo pipefail
+
+ full_ci='true'
+ reason='ordinary pull request'
+
+ if jq -e 'type == "object"' <<< "$STACK_JSON" >/dev/null 2>&1; then
+ stack_position="$(jq -r '.position // empty' <<< "$STACK_JSON")"
+ stack_size="$(jq -r '.size // empty' <<< "$STACK_JSON")"
+ stack_base_ref="$(jq -r '.base.ref // empty' <<< "$STACK_JSON")"
+
+ if [[ ! "$stack_position" =~ ^[1-9][0-9]*$ ]] ||
+ [[ ! "$stack_size" =~ ^[1-9][0-9]*$ ]] ||
+ (( stack_position > stack_size )) ||
+ [[ -z "$stack_base_ref" ]]; then
+ reason='malformed stack metadata; defaulting to full CI'
+ elif (( stack_position == stack_size )); then
+ reason='top pull request in stack'
+ elif [[ "$stack_base_ref" == "$PR_BASE_REF" ]]; then
+ reason='lowest unmerged pull request in stack'
+ else
+ full_ci='false'
+ reason='middle pull request in stack'
+ fi
+ fi
+
+ echo "full_ci=$full_ci" >> "$GITHUB_OUTPUT"
+ echo "::notice title=Stacked PR CI scope::$reason; full_ci=$full_ci"
+
policy:
needs: [gate]
runs-on: ${{ needs.gate.outputs.runner }}
@@ -323,6 +359,7 @@ jobs:
typecheck_release_registry:
name: Typecheck + Release Registry
needs: [gate, policy]
+ if: ${{ needs.gate.outputs.full_ci == 'true' }}
runs-on: ${{ needs.gate.outputs.runner }}
timeout-minutes: 20
@@ -362,6 +399,7 @@ jobs:
general_tests:
name: General tests (${{ matrix.group_label }})
needs: [gate, policy]
+ if: ${{ needs.gate.outputs.full_ci == 'true' }}
runs-on: ${{ needs.gate.outputs.runner }}
timeout-minutes: 20
strategy:
@@ -452,24 +490,41 @@ jobs:
# Preserve the legacy required-check name while the underlying work runs in parallel.
name: verify
if: ${{ always() }}
- needs: [gate, typecheck_release_registry, general_tests, build]
+ needs: [gate, policy, typecheck_release_registry, general_tests, build]
runs-on: ${{ needs.gate.outputs.runner }}
timeout-minutes: 5
steps:
- name: Fail if any split verify lane failed
env:
+ FULL_CI: ${{ needs.gate.outputs.full_ci }}
+ POLICY_RESULT: ${{ needs.policy.result }}
TYPECHECK_RELEASE_REGISTRY_RESULT: ${{ needs.typecheck_release_registry.result }}
GENERAL_TESTS_RESULT: ${{ needs.general_tests.result }}
BUILD_RESULT: ${{ needs.build.result }}
run: |
- test "$TYPECHECK_RELEASE_REGISTRY_RESULT" = "success"
- test "$GENERAL_TESTS_RESULT" = "success"
- test "$BUILD_RESULT" = "success"
+ test "$POLICY_RESULT" = "success"
+ case "$FULL_CI" in
+ true)
+ test "$TYPECHECK_RELEASE_REGISTRY_RESULT" = "success"
+ test "$GENERAL_TESTS_RESULT" = "success"
+ test "$BUILD_RESULT" = "success"
+ ;;
+ false)
+ test "$TYPECHECK_RELEASE_REGISTRY_RESULT" = "skipped"
+ test "$GENERAL_TESTS_RESULT" = "skipped"
+ test "$BUILD_RESULT" = "skipped"
+ ;;
+ *)
+ echo "Invalid full_ci decision: $FULL_CI" >&2
+ exit 1
+ ;;
+ esac
build:
name: Build
needs: [gate, policy]
+ if: ${{ needs.gate.outputs.full_ci == 'true' }}
runs-on: ${{ needs.gate.outputs.runner }}
timeout-minutes: 20
@@ -509,6 +564,7 @@ jobs:
verify_serialized_server:
name: Verify serialized server suites (${{ matrix.shard_label }})
needs: [gate, policy]
+ if: ${{ needs.gate.outputs.full_ci == 'true' }}
runs-on: ${{ needs.gate.outputs.runner }}
timeout-minutes: 20
strategy:
@@ -570,6 +626,7 @@ jobs:
canary_dry_run:
name: Canary Dry Run
needs: [gate, policy]
+ if: ${{ needs.gate.outputs.full_ci == 'true' }}
runs-on: ${{ needs.gate.outputs.runner }}
timeout-minutes: 20
@@ -627,6 +684,7 @@ jobs:
e2e_shards:
name: e2e shard (${{ matrix.shard_label }})
needs: [gate, policy]
+ if: ${{ needs.gate.outputs.full_ci == 'true' }}
runs-on: ${{ needs.gate.outputs.runner }}
timeout-minutes: 30
strategy:
@@ -728,12 +786,23 @@ jobs:
# across the matrix above (same pattern as the `verify` aggregate).
name: e2e
if: ${{ always() }}
- needs: [gate, e2e_shards]
+ needs: [gate, policy, e2e_shards]
runs-on: ${{ needs.gate.outputs.runner }}
timeout-minutes: 5
steps:
- name: Fail if any e2e shard failed
env:
+ FULL_CI: ${{ needs.gate.outputs.full_ci }}
+ POLICY_RESULT: ${{ needs.policy.result }}
E2E_SHARDS_RESULT: ${{ needs.e2e_shards.result }}
- run: test "$E2E_SHARDS_RESULT" = "success"
+ run: |
+ test "$POLICY_RESULT" = "success"
+ case "$FULL_CI" in
+ true) test "$E2E_SHARDS_RESULT" = "success" ;;
+ false) test "$E2E_SHARDS_RESULT" = "skipped" ;;
+ *)
+ echo "Invalid full_ci decision: $FULL_CI" >&2
+ exit 1
+ ;;
+ esac
diff --git a/scripts/__tests__/e2e-shard.test.mjs b/scripts/__tests__/e2e-shard.test.mjs
index 9b80a99edf..e1074ea683 100644
--- a/scripts/__tests__/e2e-shard.test.mjs
+++ b/scripts/__tests__/e2e-shard.test.mjs
@@ -1,6 +1,7 @@
import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
-import { readFileSync } from "node:fs";
+import { mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
import test from "node:test";
@@ -39,6 +40,58 @@ function readPinnedTrustedPrWorkflow() {
return result.stdout;
}
+function readWorkflowJobs(workflow) {
+ const jobs = new Map();
+ let current = null;
+ for (const line of workflow.split("\n")) {
+ const header = /^ {2}([A-Za-z0-9_-]+):\s*$/.exec(line);
+ if (header) {
+ current = header[1];
+ jobs.set(current, []);
+ continue;
+ }
+ if (current && /^\S/.test(line)) current = null;
+ if (current) jobs.get(current).push(line);
+ }
+ for (const [id, lines] of jobs) jobs.set(id, lines.join("\n"));
+ return jobs;
+}
+
+function runStackScope(stack, prBaseRef) {
+ const workflow = readFileSync(trustedPrWorkflow, "utf8");
+ const match = workflow.match(
+ / - name: Select stacked PR CI scope[\s\S]*? run: \|\n([\s\S]*?)\n\n policy:/,
+ );
+ assert.ok(match, "trusted workflow must define the stacked PR scope script");
+ const script = match[1]
+ .split("\n")
+ .map((line) => line.replace(/^ {10}/, ""))
+ .join("\n");
+ const scratch = mkdtempSync(path.join(tmpdir(), "paperclip-stack-scope-"));
+ const output = path.join(scratch, "github-output");
+
+ try {
+ const result = spawnSync("bash", ["-c", script], {
+ encoding: "utf8",
+ env: {
+ ...process.env,
+ GITHUB_OUTPUT: output,
+ PR_BASE_REF: prBaseRef,
+ STACK_JSON: JSON.stringify(stack),
+ },
+ });
+ assert.equal(result.status, 0, result.stderr);
+ return Object.fromEntries(
+ readFileSync(output, "utf8")
+ .trim()
+ .split("\n")
+ .map((line) => line.split("=")),
+ );
+ } finally {
+ rmSync(scratch, { recursive: true, force: true });
+ }
+}
+
test("the e2e shards form a complete, non-overlapping partition", () => {
const specs = listE2eSpecs();
assert.ok(specs.length > 0, "expected a non-empty e2e spec set");
@@ -115,19 +168,7 @@ test("the trusted PR workflow keeps a stable aggregate check named e2e over the
// as `e2e shard (n/3)`, so the aggregate job below is what keeps the
// required-check contract intact — same pattern as the `verify` aggregate.
const workflow = readPinnedTrustedPrWorkflow();
- const jobs = new Map();
- let current = null;
- for (const line of workflow.split("\n")) {
- const header = /^ {2}([A-Za-z0-9_-]+):\s*$/.exec(line);
- if (header) {
- current = header[1];
- jobs.set(current, []);
- continue;
- }
- if (current && /^\S/.test(line)) current = null;
- if (current) jobs.get(current).push(line);
- }
- for (const [id, lines] of jobs) jobs.set(id, lines.join("\n"));
+ const jobs = readWorkflowJobs(workflow);
const aggregate = jobs.get("e2e");
assert.ok(aggregate, "pr-trusted.yml must define an `e2e` job to satisfy branch protection");
@@ -135,8 +176,8 @@ test("the trusted PR workflow keeps a stable aggregate check named e2e over the
assert.match(aggregate, /^ {4}if: \$\{\{ always\(\) \}\}$/m, "the aggregate must run even when a shard fails");
assert.match(
aggregate,
- /^ {4}needs: \[gate, e2e_shards\]$/m,
- "the aggregate must depend on the runner gate and shard matrix",
+ /^ {4}needs: \[gate, (?:policy, )?e2e_shards\]$/m,
+ "the aggregate must depend on the runner gate, optional policy gate, and shard matrix",
);
assert.match(
aggregate,
@@ -168,6 +209,70 @@ test("the trusted PR workflow keeps a stable aggregate check named e2e over the
}
});
+test("the trusted PR workflow limits full CI to merge-relevant stack layers", () => {
+ const workflow = readFileSync(trustedPrWorkflow, "utf8");
+ const jobs = readWorkflowJobs(workflow);
+ const gate = jobs.get("gate");
+
+ assert.match(gate, /^ {6}full_ci: \$\{\{ steps\.scope\.outputs\.full_ci \}\}$/m);
+ assert.match(gate, /STACK_JSON: \$\{\{ toJSON\(github\.event\.pull_request\.stack\) \}\}/);
+ assert.match(gate, /stack_position == stack_size/);
+ assert.match(gate, /"\$stack_base_ref" == "\$PR_BASE_REF"/);
+
+ for (const jobId of [
+ "typecheck_release_registry",
+ "general_tests",
+ "build",
+ "verify_serialized_server",
+ "canary_dry_run",
+ "e2e_shards",
+ ]) {
+ assert.match(
+ jobs.get(jobId),
+ /^ {4}if: \$\{\{ needs\.gate\.outputs\.full_ci == 'true' \}\}$/m,
+ `${jobId} must run only when the gate selects full CI`,
+ );
+ }
+
+ assert.doesNotMatch(
+ jobs.get("policy"),
+ /needs\.gate\.outputs\.full_ci/,
+ "the policy job must run on every PR layer",
+ );
+
+ const verify = jobs.get("verify");
+ assert.match(verify, /^ {4}needs: \[gate, policy, typecheck_release_registry, general_tests, build\]$/m);
+ assert.match(verify, /POLICY_RESULT: \$\{\{ needs\.policy\.result \}\}/);
+ assert.match(verify, /test "\$TYPECHECK_RELEASE_REGISTRY_RESULT" = "skipped"/);
+ assert.match(verify, /test "\$GENERAL_TESTS_RESULT" = "skipped"/);
+ assert.match(verify, /test "\$BUILD_RESULT" = "skipped"/);
+
+ const e2e = jobs.get("e2e");
+ assert.match(e2e, /^ {4}needs: \[gate, policy, e2e_shards\]$/m);
+ assert.match(e2e, /POLICY_RESULT: \$\{\{ needs\.policy\.result \}\}/);
+ assert.match(e2e, /false\) test "\$E2E_SHARDS_RESULT" = "skipped"/);
+});
+
+test("the stacked PR scope selector runs full CI only where intended", () => {
+ assert.equal(runStackScope(null, "master").full_ci, "true");
+ assert.equal(
+ runStackScope({ position: 11, size: 11, base: { ref: "master" } }, "stack-10").full_ci,
+ "true",
+ );
+ assert.equal(
+ runStackScope({ position: 1, size: 11, base: { ref: "master" } }, "master").full_ci,
+ "true",
+ );
+ assert.equal(
+ runStackScope({ position: 6, size: 11, base: { ref: "master" } }, "stack-5").full_ci,
+ "false",
+ );
+ assert.equal(
+ runStackScope({ position: "invalid", size: 11, base: { ref: "master" } }, "stack-5").full_ci,
+ "true",
+ );
+});
+
test("the trusted PR workflow passes the shard's spec filter to Playwright without a literal --", () => {
// `pnpm run test:e2e -- $specs` forwards the literal separator to Playwright,
// so the specs after it are not applied as file filters.