Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

fix(ci): benchmark stacked pull requests against main (#2200)

kt1eee2938b4ee
diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index 3bc5e5a15a..14f8798415 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -3,7 +3,6 @@ run-name: 'Prime Agent benchmarks · PR #${{ github.event.pull_request.number ||
on:
pull_request_target:
- branches: [main]
types: [opened, synchronize, reopened, ready_for_review]
workflow_dispatch:
inputs:
@@ -34,6 +33,7 @@ jobs:
allowed: ${{ steps.resolve.outputs.needed == 'true' && steps.vouch.outputs.vouched == 'true' }}
harness: ${{ steps.resolve.outputs.harness }}
steps:
+ # pull_request_target supplies the default-branch SHA, including for stacked PRs.
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
ref: ${{ github.sha }}
diff --git a/scripts/benchmarks/README.md b/scripts/benchmarks/README.md
index 99487d27ae..8785bad55d 100644
--- a/scripts/benchmarks/README.md
+++ b/scripts/benchmarks/README.md
@@ -1,8 +1,9 @@
# PR performance benchmarks
-Each push to an open, vouched PR starts an informational Prime Agent benchmark. Multiple commits in
-one push produce one run for the final head. Draft PRs are included. A two-second debounce and
-per-PR cancellation avoid finishing obsolete runs. `workflow_dispatch` reruns an open PR by number.
+Each push to an open, vouched PR starts an informational Prime Agent benchmark, including PRs targeting
+another branch in a stack. Multiple commits in one push produce one run for the final head. Draft PRs
+are included. A two-second debounce and per-PR cancellation avoid finishing obsolete runs.
+`workflow_dispatch` reruns an open PR by number.
Identical automatic requests reuse the completed comment when both SHAs, harness, and configuration
match. Manual dispatch and GitHub reruns force fresh measurements. Unvouched authors
receive a pending-trust comment; a maintainer can rerun after vouching.
@@ -10,6 +11,7 @@ receive a pending-trust comment; a maintainer can rerun after vouching.
The controller resolves current `main` and the PR head to full SHAs, builds both in separate Prime
sandboxes, and alternates their measurements. Both use the same trusted harness revision, image
digest and resource allocation. No performance gate blocks merging.
+The baseline is always current `main`, not the PR's target branch or merge base.
## Enable in GitHub
@@ -21,6 +23,11 @@ The workflows must first land on `main`: both `pull_request_target` and the comp
trusted default-branch code. Trigger `Prime Agent benchmarks` manually with an open PR number after
configuring the sandbox secret. The first rollout should include a main-versus-main calibration.
+The request checks out the event's `GITHUB_SHA`, which GitHub resolves to the default-branch commit for
+[`pull_request_target`](https://github.blog/changelog/2025-11-07-actions-pull_request_target-and-environment-branch-protections-changes/),
+including stacked PRs. The benchmark job uses that exact harness SHA. Manual dispatch is allowed only
+from `main`.
+
No credentials are injected into the sandboxes. The sandbox control key and GitHub token stay on the
trusted controller; a separate publisher owns GitHub comment write permission. No PR checkout,
build script, installer, executable, or archive is executed/extracted on a privileged GitHub runner.
diff --git a/scripts/benchmarks/github.py b/scripts/benchmarks/github.py
index 3e9ff6baa0..dac11943d4 100644
--- a/scripts/benchmarks/github.py
+++ b/scripts/benchmarks/github.py
@@ -47,8 +47,8 @@ class GitHub:
self, pr: int, harness_sha: str, run_id: int, attempt: int, config: Config
) -> tuple[Report, str]:
pull = self.request("GET", f"pulls/{pr}")
- if pull["state"] != "open" or pull["base"]["ref"] != "main":
- raise ValueError("Benchmark requires an open PR targeting main")
+ if pull["state"] != "open":
+ raise ValueError("Benchmark requires an open PR")
if pull["base"]["repo"]["full_name"] != self.repository:
raise ValueError("PR belongs to another repository")
base = self.request("GET", "git/ref/heads/main")["object"]["sha"]
diff --git a/scripts/benchmarks/tests/test_benchmarks.py b/scripts/benchmarks/tests/test_benchmarks.py
index 9f5017f939..6e833071f3 100644
--- a/scripts/benchmarks/tests/test_benchmarks.py
+++ b/scripts/benchmarks/tests/test_benchmarks.py
@@ -252,6 +252,10 @@ class ReportTests(unittest.TestCase):
class FakeGitHub(GitHub):
def __init__(self):
+ self.repository = "PrimeIntellect-ai/prime-agent"
+ self.base_repository = self.head_repository = self.repository
+ self.base_ref = "main"
+ self.base_sha = self.main_sha = SHA
self.head = HEAD
self.state = "open"
self.attempt = 1
@@ -264,7 +268,18 @@ class FakeGitHub(GitHub):
self.writes.append((method, path, body))
return {}
if path == "pulls/42":
- return {"state": self.state, "head": {"sha": self.head}}
+ return {
+ "state": self.state,
+ "base": {
+ "ref": self.base_ref,
+ "sha": self.base_sha,
+ "repo": {"full_name": self.base_repository},
+ },
+ "head": {"sha": self.head, "repo": {"full_name": self.head_repository}},
+ "user": {"login": "kevin"},
+ }
+ if path == "git/ref/heads/main":
+ return {"object": {"sha": self.main_sha}}
if path == "actions/runs/100":
return {"run_attempt": self.attempt, "run_number": 10}
raise AssertionError(path)
@@ -273,6 +288,73 @@ class FakeGitHub(GitHub):
yield self.runs if key else self.comments
+class ResolverTests(unittest.TestCase):
+ def test_open_prs_use_current_main_and_exact_head_with_the_trusted_harness(self):
+ config = Config.load()
+ for base_ref in ("main", "refactor/stack-parent"):
+ for head_repository in ("PrimeIntellect-ai/prime-agent", "contributor/prime-agent"):
+ with self.subTest(base_ref=base_ref, head_repository=head_repository):
+ github = FakeGitHub()
+ github.base_ref = base_ref
+ github.base_sha = "d" * 40
+ github.head_repository = head_repository
+ report, author = github.resolve(42, "c" * 40, 100, 2, config)
+ self.assertEqual(report.repository, github.repository)
+ self.assertEqual(report.head_repository, head_repository)
+ self.assertEqual((report.pr, report.run_id, report.attempt), (42, 100, 2))
+ self.assertEqual((report.base_sha, report.main.sha), (SHA, SHA))
+ self.assertEqual((report.head_sha, report.pr_head.sha), (HEAD, HEAD))
+ self.assertEqual(report.harness_sha, "c" * 40)
+ self.assertEqual(report.config, config)
+ self.assertEqual(author, "kevin")
+ self.assertEqual(github.writes, [])
+
+ def test_closed_and_foreign_repository_prs_are_rejected_for_any_target(self):
+ for base_ref in ("main", "refactor/stack-parent"):
+ for case in ("closed", "foreign"):
+ with self.subTest(base_ref=base_ref, case=case):
+ github = FakeGitHub()
+ github.base_ref = base_ref
+ if case == "closed":
+ github.state = "closed"
+ error = "requires an open PR"
+ else:
+ github.base_repository = "other/prime-agent"
+ error = "belongs to another repository"
+ with self.assertRaisesRegex(ValueError, error):
+ github.resolve(42, "c" * 40, 100, 1, Config.load())
+
+ def test_stacked_request_pins_the_workflow_sha_not_pr_base_or_head_metadata(self):
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ event, output = root / "event.json", root / "output.txt"
+ github = FakeGitHub()
+ github.base_ref = "refactor/stack-parent"
+ github.base_sha = "d" * 40
+ pull = github.request("GET", "pulls/42") | {"number": 42}
+ event.write_text(json.dumps({"pull_request": pull}))
+ with (
+ patch("cli.GitHub", return_value=github),
+ patch.object(sys, "argv", ["cli.py", "resolve", "--results", directory]),
+ patch.dict(
+ os.environ,
+ {
+ "GITHUB_EVENT_PATH": str(event),
+ "GITHUB_OUTPUT": str(output),
+ "GITHUB_EVENT_NAME": "pull_request_target",
+ "GITHUB_SHA": "c" * 40,
+ "GITHUB_RUN_ID": "100",
+ "GITHUB_RUN_ATTEMPT": "1",
+ },
+ ),
+ ):
+ main()
+ report = load_report(root / "report.json")
+ self.assertEqual((report.harness_sha, report.base_sha, report.head_sha), ("c" * 40, SHA, HEAD))
+ self.assertEqual(output.read_text(), f"pr=42\nauthor=kevin\nharness={'c' * 40}\nneeded=true\n")
+ self.assertEqual(github.writes, [])
+
+
class PublishingTests(unittest.TestCase):
def test_new_commit_replaces_the_previous_table_in_the_same_comment(self):
previous = fixture()
@@ -304,6 +386,7 @@ class PublishingTests(unittest.TestCase):
for case in ("head", "new_run", "attempt", "closed"):
with self.subTest(case=case):
github = FakeGitHub()
+ github.base_ref = "refactor/stack-parent"
if case == "head":
github.head = SHA
elif case == "new_run":
@@ -481,6 +564,7 @@ class LifecycleTests(unittest.TestCase):
"path": ".github/workflows/benchmarks.yml",
"repository": {"full_name": "PrimeIntellect-ai/prime-agent"},
"head_repository": {"full_name": "contributor/prime-agent"},
+ "head_branch": "refactor/stack-child",
"head_sha": HEAD,
"id": 100,
"run_attempt": 1,