Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

[Eval] Add pre-release behavioral evaluation CI (#2306)

Seth Kartenc13e03113dfa
diff --git a/.github/workflows/behavioral-evals.yml b/.github/workflows/behavioral-evals.yml
new file mode 100644
index 0000000000..3af2c96797
--- /dev/null
+++ b/.github/workflows/behavioral-evals.yml
@@ -0,0 +1,491 @@
+name: Behavioral Eval
+run-name: >-
+ Behavioral Eval · ${{ github.event_name == 'push' && 'base update' ||
+ format('PR #{0}', github.event.pull_request.number) }}
+
+on:
+ pull_request_target:
+ branches: [main]
+ types: [labeled, unlabeled, synchronize, edited, reopened]
+ push:
+ branches: [main]
+
+concurrency:
+ group: >-
+ ${{
+ github.event_name == 'pull_request_target' &&
+ ((github.event.action == 'labeled' && github.event.label.name == 'pre-release') ||
+ (github.event.action == 'unlabeled' && github.event.label.name == 'pre-release') ||
+ ((github.event.action == 'synchronize' || github.event.action == 'reopened' ||
+ (github.event.action == 'edited' && github.event.changes.base)) &&
+ contains(github.event.pull_request.labels.*.name, 'pre-release'))) &&
+ format('prime-agent-behavioral-{0}', github.event.pull_request.number) ||
+ format('prime-agent-behavioral-skip-{0}', github.run_id)
+ }}
+ cancel-in-progress: true
+
+permissions: {}
+
+jobs:
+ invalidate-on-pr-change:
+ if: >-
+ github.event_name == 'pull_request_target' &&
+ ((github.event.action == 'unlabeled' && github.event.label.name == 'pre-release') ||
+ ((github.event.action == 'synchronize' || github.event.action == 'reopened' ||
+ (github.event.action == 'edited' && github.event.changes.base)) &&
+ contains(github.event.pull_request.labels.*.name, 'pre-release')))
+ runs-on: ubuntu-24.04
+ permissions:
+ statuses: write
+ steps:
+ - name: Revoke the durable release status
+ uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v8.0.0
+ with:
+ script: |
+ const action = context.payload.action;
+ const description = action === 'unlabeled'
+ ? 'pre-release approval removed'
+ : action === 'synchronize'
+ ? 'Head changed; reapply pre-release'
+ : 'PR identity changed; reapply pre-release';
+ for (const statusContext of [
+ 'Behavioral Eval / pre-release approval',
+ 'Behavioral Eval / pre-release',
+ ]) {
+ await github.rest.repos.createCommitStatus({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ sha: context.payload.pull_request.head.sha,
+ state: 'failure',
+ context: statusContext,
+ description,
+ target_url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`,
+ });
+ }
+
+ invalidate-on-base-change:
+ if: github.event_name == 'push'
+ runs-on: ubuntu-24.04
+ permissions:
+ pull-requests: read
+ statuses: write
+ steps:
+ - name: Revoke results bound to the previous base
+ uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v8.0.0
+ with:
+ script: |
+ const pulls = await github.paginate(github.rest.pulls.list, {
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ state: 'open',
+ base: context.ref.replace('refs/heads/', ''),
+ per_page: 100,
+ });
+ for (const pull of pulls) {
+ const statuses = await github.paginate(github.rest.repos.listCommitStatusesForRef, {
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ ref: pull.head.sha,
+ per_page: 100,
+ });
+ const contexts = [
+ 'Behavioral Eval / pre-release approval',
+ 'Behavioral Eval / pre-release',
+ ];
+ if (!statuses.some(status => contexts.includes(status.context))) continue;
+ for (const statusContext of contexts) {
+ await github.rest.repos.createCommitStatus({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ sha: pull.head.sha,
+ state: 'failure',
+ context: statusContext,
+ description: 'Base advanced; reapply pre-release',
+ target_url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`,
+ });
+ }
+ }
+
+ evaluate:
+ if: >-
+ github.event_name == 'pull_request_target' && github.event.action == 'labeled' &&
+ github.event.label.name == 'pre-release'
+ name: runner
+ runs-on: ubuntu-24.04
+ timeout-minutes: 360
+ permissions:
+ actions: read
+ contents: read
+ statuses: write
+ steps:
+ - name: Record the job budget
+ # Mirrors timeout-minutes above so the orchestrator fits its waits inside the job.
+ run: echo "BEHAVIORAL_JOB_DEADLINE_EPOCH=$(( $(date +%s) + 360 * 60 ))" >> "$GITHUB_ENV"
+
+ - name: Mark the requested head pending
+ uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v8.0.0
+ with:
+ script: |
+ await github.rest.repos.createCommitStatus({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ sha: context.payload.pull_request.head.sha,
+ state: 'pending',
+ context: 'Behavioral Eval / pre-release',
+ description: 'Starting exact base and head validation',
+ target_url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`,
+ });
+ await github.rest.repos.createCommitStatus({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ sha: context.payload.pull_request.head.sha,
+ state: 'success',
+ context: 'Behavioral Eval / pre-release approval',
+ description: 'pre-release label approves this exact head',
+ target_url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`,
+ });
+
+ - name: Check out trusted evaluator
+ uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+ with:
+ ref: ${{ github.sha }}
+ persist-credentials: false
+
+ - name: Resolve exact base and head
+ id: resolve
+ run: >-
+ python3 scripts/evals/short_swe/ci.py
+ --output request --harness-sha "$(git rev-parse HEAD)"
+
+ - name: Set up uv
+ uses: astral-sh/setup-uv@94527f2e458b27549849d47d273a16bec83a01e9 # v7
+ with:
+ version: 0.12.9
+ enable-cache: false
+
+ - name: Install controller tooling
+ run: uv sync --locked --project scripts/benchmarks
+
+ - name: Check out pinned Verifiers
+ uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+ with:
+ repository: PrimeIntellect-ai/verifiers
+ ref: 9df6a3c01bb640c34380f4f6dff63a1b168dee22
+ path: vendor/verifiers
+ persist-credentials: false
+
+ - name: Check out pinned environments
+ uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+ with:
+ repository: PrimeIntellect-ai/prime-envs
+ ref: a41660014674ba43d56ed054199dde25ddf69946
+ path: vendor/environments
+ persist-credentials: false
+
+ - name: Install pinned evaluator
+ id: evaluator
+ continue-on-error: true
+ run: |
+ uv sync --locked --project vendor/verifiers --extra harbor
+ uv pip install --python vendor/verifiers/.venv/bin/python --no-deps \
+ -e vendor/environments/environments/swe/swebench_verified \
+ -e vendor/environments/environments/swe/swebench_pro \
+ -e vendor/environments/environments/swe/scaleswe
+
+ - name: Build exact base and head in isolated sandboxes
+ id: build
+ continue-on-error: true
+ if: steps.evaluator.outcome == 'success'
+ run: |
+ runner=(uv run --locked --project scripts/benchmarks python scripts/evals/short_swe/build_controller.py)
+ "${runner[@]}" --repository "$GITHUB_REPOSITORY" --source-repository "$GITHUB_REPOSITORY" \
+ --sha "${{ steps.resolve.outputs.base_sha }}" --side base \
+ --run-id "$GITHUB_RUN_ID" --attempt "$GITHUB_RUN_ATTEMPT" --output artifacts/base &
+ base_pid=$!
+ "${runner[@]}" --repository "$GITHUB_REPOSITORY" \
+ --source-repository "${{ steps.resolve.outputs.head_repository }}" \
+ --sha "${{ steps.resolve.outputs.head_sha }}" --side head \
+ --run-id "$GITHUB_RUN_ID" --attempt "$GITHUB_RUN_ATTEMPT" --output artifacts/head &
+ head_pid=$!
+ status=0
+ wait "$base_pid" || status=1
+ wait "$head_pid" || status=1
+ exit "$status"
+ env:
+ PRIME_SANDBOX_API_KEY: ${{ secrets.PRIME_SANDBOX_API_KEY }}
+ PRIME_TEAM_ID: ${{ vars.PRIME_BENCHMARK_TEAM_ID }}
+
+ - name: Generate the fixed 15/8/5 configs
+ id: prepare
+ continue-on-error: true
+ if: steps.build.outcome == 'success'
+ run: |
+ python3 scripts/evals/short_swe/prepare.py \
+ --verifiers vendor/verifiers --environments vendor/environments \
+ --base-artifacts artifacts/base --base-sha "${{ steps.resolve.outputs.base_sha }}" \
+ --head-artifacts artifacts/head --head-sha "${{ steps.resolve.outputs.head_sha }}" \
+ --output generated-configs
+
+ - name: Run the deterministic gold-patch oracle
+ id: oracle
+ continue-on-error: true
+ if: steps.prepare.outcome == 'success'
+ run: |
+ vendor/verifiers/.venv/bin/python scripts/evals/short_swe/evaluate.py \
+ --eval vendor/verifiers/.venv/bin/eval --configs generated-configs \
+ --output oracle-eval --request request/request.json --result results/oracle.json \
+ --oracle-only
+ env:
+ PRIME_API_KEY: ${{ secrets.PRIME_BEHAVIORAL_API_KEY }}
+ PRIME_SANDBOX_API_KEY: ${{ secrets.PRIME_SANDBOX_API_KEY }}
+ PRIME_TEAM_ID: ${{ vars.PRIME_BENCHMARK_TEAM_ID }}
+
+ - name: Publish the base candidate tarballs
+ id: upload-base
+ continue-on-error: true
+ if: steps.oracle.outcome == 'success'
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: candidate-tarballs-base
+ path: artifacts/base/*.tgz
+ if-no-files-found: error
+ retention-days: 7
+
+ - name: Publish the head candidate tarballs
+ id: upload-head
+ continue-on-error: true
+ if: steps.upload-base.outcome == 'success'
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: candidate-tarballs-head
+ path: artifacts/head/*.tgz
+ if-no-files-found: error
+ retention-days: 7
+
+ - name: Build the hosted candidate sources
+ id: sources
+ continue-on-error: true
+ if: steps.upload-head.outcome == 'success'
+ run: |
+ python3 - <<'PY'
+ import json
+ import os
+ from pathlib import Path
+
+ def side(name: str, sha: str, artifact_id: str) -> dict:
+ manifest = json.loads(Path(f"artifacts/{name}/artifact-manifest.json").read_text())
+ if manifest["sha"] != sha:
+ raise ValueError(f"{name}: artifact manifest does not match the request")
+ return {
+ "tarballs_url": (
+ f"https://api.github.com/repos/{os.environ['GITHUB_REPOSITORY']}"
+ f"/actions/artifacts/{artifact_id}/zip"
+ ),
+ "commit": sha,
+ "checksums": {row["name"]: row["sha256"] for row in manifest["artifacts"]},
+ "token": os.environ["GH_TOKEN"],
+ }
+
+ request = json.loads(Path("request/request.json").read_text())
+ sources = {
+ "base": side("base", request["base_sha"], "${{ steps.upload-base.outputs.artifact-id }}"),
+ "head": side("head", request["head_sha"], "${{ steps.upload-head.outputs.artifact-id }}"),
+ }
+ Path("request/sources.json").write_text(json.dumps(sources))
+ PY
+ env:
+ GH_TOKEN: ${{ github.token }}
+
+ - name: Install the Prime CLI
+ id: prime-cli
+ continue-on-error: true
+ if: steps.sources.outcome == 'success'
+ run: uv tool install prime==0.6.31
+
+ - name: Launch the six hosted evaluations
+ id: hosted
+ continue-on-error: true
+ if: steps.prime-cli.outcome == 'success'
+ run: |
+ uv run --locked --project scripts/benchmarks python \
+ scripts/evals/short_swe/hosted_eval.py launch \
+ --request request/request.json --sources request/sources.json --output hosted
+ env:
+ PRIME_API_KEY: ${{ secrets.PRIME_BEHAVIORAL_API_KEY }}
+ PRIME_TEAM_ID: ${{ vars.PRIME_BENCHMARK_TEAM_ID }}
+
+ - name: Wait for the hosted evaluations
+ id: hosted-wait
+ continue-on-error: true
+ if: steps.hosted.outcome == 'success'
+ run: |
+ uv run --locked --project scripts/benchmarks python \
+ scripts/evals/short_swe/hosted_eval.py wait --output hosted
+ env:
+ PRIME_API_KEY: ${{ secrets.PRIME_BEHAVIORAL_API_KEY }}
+
+ - name: Collect the hosted episodes
+ id: hosted-collect
+ continue-on-error: true
+ if: steps.hosted-wait.outcome == 'success'
+ run: |
+ uv run --locked --project scripts/benchmarks python \
+ scripts/evals/short_swe/hosted_eval.py collect --output hosted
+ env:
+ PRIME_API_KEY: ${{ secrets.PRIME_BEHAVIORAL_API_KEY }}
+
+ - name: Gate the paired hosted result
+ id: evaluate
+ continue-on-error: true
+ if: steps.hosted-collect.outcome == 'success'
+ run: |
+ vendor/verifiers/.venv/bin/python scripts/evals/short_swe/evaluate.py \
+ --eval vendor/verifiers/.venv/bin/eval --configs generated-configs \
+ --output hosted/raw-eval --request request/request.json \
+ --result results/paired.json --from-existing
+
+ - name: Stop hosted evaluations and clean up sandboxes
+ if: always()
+ continue-on-error: true
+ run: |
+ uv run --locked --project scripts/benchmarks python \
+ scripts/evals/short_swe/hosted_eval.py stop --output hosted
+ uv run --locked --project scripts/benchmarks python scripts/evals/short_swe/cleanup.py \
+ --repository "$GITHUB_REPOSITORY" --run-id "$GITHUB_RUN_ID" \
+ --attempt "$GITHUB_RUN_ATTEMPT"
+ env:
+ PRIME_API_KEY: ${{ secrets.PRIME_BEHAVIORAL_API_KEY }}
+ PRIME_SANDBOX_API_KEY: ${{ secrets.PRIME_SANDBOX_API_KEY }}
+ PRIME_TEAM_ID: ${{ vars.PRIME_BENCHMARK_TEAM_ID }}
+
+ - name: Render trusted report
+ if: always()
+ run: |
+ python3 scripts/evals/short_swe/report.py \
+ --request request/request.json --result results/paired.json \
+ --markdown results/report.md --verdict results/verdict \
+ --evaluations hosted/hosted-runs.json
+ cat results/report.md >> "$GITHUB_STEP_SUMMARY"
+
+ - name: Save packages, traces, and paired results
+ if: always()
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: behavioral-results-${{ github.run_attempt }}
+ path: |
+ artifacts
+ generated-configs
+ oracle-eval
+ hosted
+ request
+ !request/sources.json
+ results
+ if-no-files-found: warn
+ retention-days: 30
+
+ - name: Set exact-head status
+ if: always()
+ uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v8.0.0
+ with:
+ script: |
+ const fs = require('node:fs');
+ let request;
+ try {
+ request = JSON.parse(fs.readFileSync('request/request.json', 'utf8'));
+ } catch {
+ request = null;
+ }
+ const {data: pull} = await github.rest.pulls.get({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ pull_number: context.payload.pull_request.number,
+ });
+ const rules = await github.paginate(
+ 'GET /repos/{owner}/{repo}/rules/branches/{branch}',
+ {
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ branch: pull.base.ref,
+ per_page: 100,
+ },
+ );
+ const requiredContexts = new Set([
+ 'Behavioral Eval / pre-release approval',
+ 'Behavioral Eval / pre-release',
+ ]);
+ const strictContexts = new Set(
+ rules
+ .filter(rule => rule.type === 'required_status_checks' &&
+ rule.parameters?.strict_required_status_checks_policy === true)
+ .flatMap(rule => rule.parameters.required_status_checks.map(check => check.context)),
+ );
+ const strictRequired = [...requiredContexts].every(
+ contextName => strictContexts.has(contextName),
+ );
+ const labelPresent = pull.labels.some(label => label.name === 'pre-release');
+ const identityCurrent = request &&
+ request.head_sha === pull.head.sha && request.base_sha === pull.base.sha;
+ const passed = strictRequired && identityCurrent && labelPresent &&
+ fs.existsSync('results/verdict') &&
+ fs.readFileSync('results/verdict', 'utf8').trim() === 'pass';
+ await github.rest.repos.createCommitStatus({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ sha: context.payload.pull_request.head.sha,
+ state: passed ? 'success' : 'failure',
+ context: 'Behavioral Eval / pre-release',
+ description: passed ? 'Paired Short SWE gate passed' : 'Paired Short SWE gate failed or stale',
+ target_url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`,
+ });
+
+ - name: Enforce complete paired result and regression thresholds
+ if: always()
+ run: test -f results/verdict && test "$(cat results/verdict)" = pass
+
+ report:
+ name: Publish PR report
+ needs: evaluate
+ if: always() && needs.evaluate.result != 'skipped'
+ runs-on: ubuntu-24.04
+ permissions:
+ contents: read
+ pull-requests: write
+ steps:
+ - name: Download evaluation results
+ uses: actions/download-artifact@642e1d0bd1d03f3d1c2f94b1f3ae300c4d0b1b4a # v7
+ with:
+ name: behavioral-results-${{ github.run_attempt }}
+ path: .
+
+ - name: Publish PR report
+ if: always() && hashFiles('results/report.md') != ''
+ uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v8.0.0
+ with:
+ script: |
+ const fs = require('node:fs');
+ const marker = '<!-- prime-agent-behavioral-eval:v1 -->';
+ const body = fs.readFileSync('results/report.md', 'utf8');
+ const comments = await github.paginate(github.rest.issues.listComments, {
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ issue_number: context.issue.number,
+ per_page: 100,
+ });
+ const prior = comments.find(comment =>
+ comment.user.type === 'Bot' && comment.body?.includes(marker));
+ if (prior) {
+ await github.rest.issues.updateComment({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ comment_id: prior.id,
+ body,
+ });
+ } else {
+ await github.rest.issues.createComment({
+ owner: context.repo.owner,
+ repo: context.repo.repo,
+ issue_number: context.issue.number,
+ body,
+ });
+ }
+
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 462dff46a3..a4b4160d95 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -158,6 +158,13 @@ jobs:
uv run --locked ruff format --check .
uv run --locked python -m unittest discover -s tests -v
+ - name: Test behavioral evaluation harness
+ if: matrix.name == 'runtime python'
+ run: |
+ uv run --locked --project scripts/benchmarks ruff check --config scripts/benchmarks/pyproject.toml scripts/evals/short_swe
+ uv run --locked --project scripts/benchmarks ruff format --config scripts/benchmarks/pyproject.toml --check scripts/evals/short_swe
+ uv run --locked --project scripts/benchmarks pytest -q scripts/evals/short_swe/tests
+
standalone:
needs: trust
if: needs.trust.outputs.allowed == 'true'
diff --git a/.gitignore b/.gitignore
index e2a17696bf..4c062218aa 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,6 +11,7 @@ packages/*/dist-firefox/
# Environment
.env
+.prime/
# Editor files
.vscode/
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index cb693bd65b..1f7e856139 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -40,6 +40,17 @@ If a maintainer has invited a pull request:
Development setup and commands are documented in the [development guide](packages/coding-agent/docs/development.md).
+## Pre-release behavioral evaluation
+
+Maintainers may apply the exact `pre-release` label to a release-candidate pull request.
+This approves the current head for a fixed Short SWE comparison with its exact base.
+Remove and reapply the label after every new head revision or base advancement. Removing
+the label revokes any prior result. Do not add it for ordinary CI. A labeled PR must pass
+the current `Behavioral Eval / pre-release approval` and `Behavioral Eval / pre-release`
+statuses before merge. Repository rules must require both contexts with strict up-to-date-branch
+enforcement; the workflow refuses evaluation success otherwise. See
+[`scripts/evals/short_swe/README.md`](scripts/evals/short_swe/README.md).
+
## Changelog entries
Do not edit `packages/*/CHANGELOG.md` directly. Instead, add one fragment
diff --git a/SECURITY.md b/SECURITY.md
index 13bf794d1c..ddb6dc6577 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -16,6 +16,18 @@ Include the following when possible:
Do not include real API keys, tokens, personal data, or credentials in the report. Use redacted or disposable test values.
+## Behavioral release evaluation
+
+The `pre-release` label enables a trusted behavioral evaluation before release. Exact base
+and head revisions build only inside isolated Prime sandboxes. GitHub runners treat their
+packages as opaque bytes and never execute or extract them. Model and sandbox credentials
+stay behind trusted Verifiers interception and are removed from candidate process
+environments. Separate durable approval and evaluation statuses prevent an in-flight evaluation
+from restoring approval after the label is removed. Both statuses are revoked when either candidate
+revision changes, and repository rules must require both with strict up-to-date enforcement. See
+[`scripts/evals/short_swe/README.md`](scripts/evals/short_swe/README.md)
+for the full boundary.
+
## What to Expect
Maintainers will assess the report, determine its scope, and coordinate remediation and disclosure when appropriate. Please allow time for investigation before publishing details that could put users at risk.
diff --git a/scripts/benchmarks/pyproject.toml b/scripts/benchmarks/pyproject.toml
index 1453224541..455781e3d3 100644
--- a/scripts/benchmarks/pyproject.toml
+++ b/scripts/benchmarks/pyproject.toml
@@ -11,7 +11,7 @@ dependencies = [
]
[dependency-groups]
-dev = ["ruff==0.16.5"]
+dev = ["pytest==8.4.2", "ruff==0.16.5"]
[tool.uv]
package = false
diff --git a/scripts/benchmarks/uv.lock b/scripts/benchmarks/uv.lock
index efb86effe0..77821bead3 100644
--- a/scripts/benchmarks/uv.lock
+++ b/scripts/benchmarks/uv.lock
@@ -45,6 +45,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" },
]
+[[package]]
+name = "colorama"
+version = "0.4.6"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" },
+]
+
[[package]]
name = "connectrpc"
version = "0.11.1"
@@ -104,6 +113,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/57/b0/0e52c878c53f245edd3a11020f20979b3f490f245af532c7cae3027754b5/idna-3.19-py3-none-any.whl", hash = "sha256:815e7be7a7806d54abb586dc943addc79e8b2ee16915059658cbeff4b1b43bf4", size = 68550, upload-time = "2026-08-18T05:14:22.343Z" },
]
+[[package]]
+name = "iniconfig"
+version = "2.3.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" },
+]
+
[[package]]
name = "opentelemetry-api"
version = "1.44.0"
@@ -116,6 +134,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/ca/6f/a04e900f465ff3221ccc395522503e2d10e79fa21f2723c8e177aae1e0d1/opentelemetry_api-1.44.0-py3-none-any.whl", hash = "sha256:94b98c893a91b88657eaac1e3ba89618cdb85be6918196705354f34728b2cdef", size = 60018, upload-time = "2026-07-16T15:25:11.657Z" },
]
+[[package]]
+name = "packaging"
+version = "26.3"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" },
+]
+
[[package]]
name = "pexpect"
version = "4.9.0"
@@ -128,6 +155,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl", hash = "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523", size = 63772, upload-time = "2023-11-25T06:56:14.81Z" },
]
+[[package]]
+name = "pluggy"
+version = "1.6.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" },
+]
+
[[package]]
name = "prime-agent-benchmarks"
version = "0.1.0"
@@ -142,6 +178,7 @@ dependencies = [
[package.dev-dependencies]
dev = [
+ { name = "pytest" },
{ name = "ruff" },
]
@@ -155,7 +192,10 @@ requires-dist = [
]
[package.metadata.requires-dev]
-dev = [{ name = "ruff", specifier = "==0.16.5" }]
+dev = [
+ { name = "pytest", specifier = "==8.4.2" },
+ { name = "ruff", specifier = "==0.16.5" },
+]
[[package]]
name = "prime-sandboxes"
@@ -364,6 +404,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/db/1d/068464f23075f66a8f1b806935e9cd9363ee446636ea70d2c22ee8659dbf/pydantic_core-2.46.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d0a24b40877af2de4950252be9d21eaf7fb07660f3c2cae1f56c6b599ada5266", size = 2140686, upload-time = "2026-08-28T10:01:28.947Z" },
]
+[[package]]
+name = "pygments"
+version = "2.21.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" },
+]
+
[[package]]
name = "pyqwest"
version = "0.10.0"
@@ -429,6 +478,22 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/59/d0/bb522283b90853afbf506cd5b71c650cf708829914efd0003d615cf426cd/pyte-0.8.2-py3-none-any.whl", hash = "sha256:85db42a35798a5aafa96ac4d8da78b090b2c933248819157fc0e6f78876a0135", size = 31627, upload-time = "2023-11-12T09:33:41.096Z" },
]
+[[package]]
+name = "pytest"
+version = "8.4.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "colorama", marker = "sys_platform == 'win32'" },
+ { name = "iniconfig" },
+ { name = "packaging" },
+ { name = "pluggy" },
+ { name = "pygments" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/a3/5c/00a0e072241553e1a7496d638deababa67c5058571567b92a7eaa258397c/pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01", size = 1519618, upload-time = "2025-09-04T14:34:22.711Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750, upload-time = "2025-09-04T14:34:20.226Z" },
+]
+
[[package]]
name = "ruff"
version = "0.16.5"
diff --git a/scripts/evals/README.md b/scripts/evals/README.md
index 4ed8d1270b..94336d1082 100644
--- a/scripts/evals/README.md
+++ b/scripts/evals/README.md
@@ -2,10 +2,17 @@
These harnesses measure end-to-end agent capability with a real model.
They complement the repo's unit tests (which never call a model) and the
-PR performance benchmarks (which measure startup, not capability). They
-never run in CI and never need credentials to validate: every harness is
-exercised by model-free self-tests, and a real-model run is a manual
-step.
+PR performance benchmarks (which measure startup, not capability). Every
+harness has model-free self-tests. Real-model runs are manual unless a harness
+documents a narrow release gate, such as the label-gated Short SWE check.
+
+
+## short_swe
+
+The Short SWE release check compares the exact base and head of a pull request on
+fixed 15/8/5 slices of SWE-bench Verified, SWE-bench Pro, and ScaleSWE. It runs
+only when a maintainer applies the exact `pre-release` label. See
+[`short_swe/README.md`](short_swe/README.md) for its trust boundary and commands.
## swarm-fanout
diff --git a/scripts/evals/short_swe/README.md b/scripts/evals/short_swe/README.md
new file mode 100644
index 0000000000..84c3c2c6f5
--- /dev/null
+++ b/scripts/evals/short_swe/README.md
@@ -0,0 +1,82 @@
+# Short SWE pre-release check
+
+This evaluation compares the exact PR base and exact PR head on 28 fixed tasks:
+15 SWE-bench Verified, 8 SWE-bench Pro, and 5 ScaleSWE. It uses
+`internal/glm-5.3-fast` with `autonomous = false`.
+
+The fixed sample is repository-stratified rather than a prefix of dataset order. Verified
+covers all 12 source repositories with a 6/7/1/1 split across the published difficulty
+buckets (`<15 min`, `15 min–1 hour`, `1–4 hours`, and `>4 hours`). Pro uses one task from
+each of eight repositories. Pro repositories and eligible tasks within each declared
+repository/difficulty bucket were ranked with the fixed seed `prime-agent-short-swe-v2`.
+Eligibility requires the pinned verifier tests to complete without external network access;
+prompts and model outcomes were not inspected when choosing between eligible tasks.
+
+The paid evaluation runs only for a `pull_request_target` `labeled` event whose label is
+exactly `pre-release`. The workflow marks the requested head pending before checkout and records
+label approval in a separate status that the evaluation finisher never writes. Removing the label
+revokes approval even if an in-flight evaluation later finishes. Head or default-branch changes
+revoke both statuses. Evaluation success also requires repository rules to require both statuses
+with strict up-to-date-branch enforcement, which closes races with a concurrent base update.
+Reapply the label to approve the new exact comparison.
+
+## Trust boundary
+
+- The workflow and evaluator come from the trusted base branch.
+- Exact base and head source revisions build as an unprivileged user in separate
+ Prime sandboxes.
+- The runner copies only bounded opaque npm tarballs from a root-owned snapshot.
+ It never extracts or executes candidate packages.
+- Verifiers uploads those packages into isolated task sandboxes. Candidate code
+ receives no GitHub, provider, or sandbox credentials.
+- The three tasksets and both comparison sides run as six private hosted evaluations on
+ Prime Evals, launched concurrently. Each evaluation runs the published `short-swe-*`
+ Environments Hub packages, which embed the fixed slices, pinned datasets, evaluation
+ limits, and the candidate harness as package defaults. Each evaluation starts at most
+ four root episodes, so the six-way launch has at most 24 root agents against the global
+ provider concurrency limit of 32 shared across every session. Recursive subagents remain
+ unrestricted; there is no shared client-side queue or semaphore. A terminal rollout
+ timeout with any provider call error fails closed.
+- The suite pins `internal/glm-5.3-fast` with `internal/deepseek-v4.1-flash` as its
+ authorized backup for a saturated global limit. Every paired task must use exactly one
+ pinned model, and base and head must use the same model; the gates fail otherwise.
+- The candidate npm tarballs are delivered to each hosted evaluation through
+ `CANDIDATE_TARBALLS_URL`, `CANDIDATE_COMMIT`, and `CANDIDATE_CHECKSUMS` secrets. The
+ trusted harness downloads and checksum-verifies them before the agent starts, and those
+ values never reach a candidate-controlled runtime. The platform's verifiers runtime is
+ version-managed by Prime Evals; the packages accept `verifiers[harbor]>=0.3.1` and the
+ gates validate every pulled episode with the pinned local evaluator.
+- Hosted episodes are pulled back with `prime eval samples`; each sample carries the full
+ native episode record, which the pinned evaluator validates with the same fail-closed
+ gates as the local paired flow. The report links every hosted evaluation for durable
+ evidence.
+- Typed Verifiers `WireTrace` episodes provide rewards, usage, timing, and task
+ identity. Missing or malformed episodes fail. Exact rollout deadlines and deterministic
+ provider rejections remain unresolved model outcomes; terminal 5xx provider outages count as unresolved model outcomes bounded by the model-failure threshold.
+- SWE-bench Verified transfers only a bounded binary source diff into a fresh, credential-free,
+ network-free verifier sandbox. The trusted evaluator parses bounded controller-captured test
+ output against pinned task metadata. The solver never receives the task package metadata; gold
+ source patches and expected statuses are removed from the verifier before repository tests run.
+ The fixed pure-Python slice uses dependencies already pinned in
+ each task image, so scoring does not resolve packages from the network. A fixed gold-patch oracle must resolve
+ before any paired task starts. Missing or inconsistent verifier output fails as infrastructure
+ rather than becoming a zero reward.
+- There is no durable baseline, promotion job, focused confirmation, or automatic
+ retry. The one run compares head directly with its exact base and never merges.
+
+## Gate
+
+The check fails if it cannot validate all 28 paired tasks. It also fails for a
+loss of at least five resolved tasks, three additional model failures, or a 2x
+output-token or cumulative-task-time increase without a resolution gain. Cumulative task time
+sums task traces, including tasks that overlap in wall-clock time. Smaller changes remain visible
+in the report. Meaningful token changes (20% or more) use green for reductions and red for
+increases.
+
+## Model-free validation
+
+```sh
+uv run --locked --project scripts/benchmarks ruff check --config scripts/benchmarks/pyproject.toml scripts/evals/short_swe
+uv run --locked --project scripts/benchmarks ruff format --config scripts/benchmarks/pyproject.toml --check scripts/evals/short_swe
+uv run --locked --project scripts/benchmarks pytest -q scripts/evals/short_swe/tests
+```
diff --git a/scripts/evals/short_swe/build_controller.py b/scripts/evals/short_swe/build_controller.py
new file mode 100644
index 0000000000..c9a1158ab7
--- /dev/null
+++ b/scripts/evals/short_swe/build_controller.py
@@ -0,0 +1,234 @@
+#!/usr/bin/env python3
+"""Build a PR revision in an isolated Prime sandbox and collect opaque tarballs."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import re
+import shlex
+import time
+from pathlib import Path
+
+from prime_sandboxes import APIClient, CreateSandboxRequest, SandboxClient
+
+ROOT = Path(__file__).resolve().parent
+REMOTE = "/opt/behavioral-build"
+OWNER_LABEL = "prime-agent-behavioral-v1"
+SHA_RE = re.compile(r"^[0-9a-f]{40}$")
+REPO_RE = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$")
+VERSION = "0.0.0-benchmark"
+EXPECTED = {
+ f"prime-agent-{VERSION}.tgz",
+ f"prime-agent-ai-{VERSION}.tgz",
+ f"prime-agent-core-{VERSION}.tgz",
+ f"prime-agent-tui-{VERSION}.tgz",
+}
+MAX_ARTIFACT_BYTES = 20_000_000
+BUILD_SANDBOX_TIMEOUT_MINUTES = 120
+PROVISION_TIMEOUT_SECONDS = 300
+SETUP_TIMEOUT_SECONDS = 180
+MAX_BUILD_SECONDS = 4_800
+CONTROL_COMMAND_TIMEOUT_SECONDS = 30
+COMMAND_TIMEOUT_LIMIT_SECONDS = 900
+BUILD_POLL_SECONDS = 10
+
+
+def labels(repository: str, run_id: int, attempt: int, side: str) -> list[str]:
+ return [
+ OWNER_LABEL,
+ f"repository:{repository}",
+ f"run:{run_id}",
+ f"attempt:{attempt}",
+ f"role:builder-{side}",
+ ]
+
+
+def wait_running(
+ client: SandboxClient,
+ sandbox_id: str,
+ timeout: int = PROVISION_TIMEOUT_SECONDS,
+) -> None:
+ deadline = time.monotonic() + timeout
+ while time.monotonic() < deadline:
+ status = client.get(sandbox_id).status
+ if status == "RUNNING":
+ return
+ if status in {"ERROR", "FAILED", "TERMINATED", "STOPPED", "TIMEOUT"}:
+ raise RuntimeError("candidate build sandbox failed during provisioning")
+ time.sleep(2)
+ raise TimeoutError("candidate build sandbox provisioning timed out")
+
+
+def run_builder(client: SandboxClient, sandbox_id: str, command: str) -> int:
+ results = f"{REMOTE}/results"
+ exit_path = f"{results}/build.exit"
+ controller_log = f"{results}/controller.log"
+ script = f"{command}; code=$?; printf '%s\\n' \"$code\" > {exit_path}.tmp; mv {exit_path}.tmp {exit_path}"
+ deadline = time.monotonic() + MAX_BUILD_SECONDS
+ start = client.execute_command(
+ sandbox_id,
+ f"mkdir -p {results}; rm -f {exit_path} {exit_path}.tmp; "
+ f"nohup sh -c {shlex.quote(script)} > {controller_log} 2>&1 </dev/null &",
+ timeout=CONTROL_COMMAND_TIMEOUT_SECONDS,
+ )
+ if start.exit_code:
+ raise RuntimeError("candidate release build could not start")
+
+ while True:
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ return 124
+ status = client.execute_command(
+ sandbox_id,
+ f"if test -f {exit_path}; then cat {exit_path}; else echo running; fi",
+ timeout=min(CONTROL_COMMAND_TIMEOUT_SECONDS, max(1, int(remaining))),
+ )
+ if status.exit_code:
+ raise RuntimeError("candidate release build status check failed")
+ value = status.stdout.strip()
+ if value != "running":
+ if not re.fullmatch(r"[0-9]{1,3}", value):
+ raise RuntimeError("candidate release build returned an invalid status")
+ return int(value)
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ return 124
+ time.sleep(min(BUILD_POLL_SECONDS, remaining))
+
+
+def build(
+ repository: str,
+ source_repository: str,
+ sha: str,
+ run_id: int,
+ attempt: int,
+ output: Path,
+ side: str,
+) -> None:
+ if side not in {"base", "head"}:
+ raise ValueError("side must be base or head")
+ if not REPO_RE.fullmatch(repository) or not REPO_RE.fullmatch(source_repository):
+ raise ValueError("invalid repository")
+ if not SHA_RE.fullmatch(sha):
+ raise ValueError("invalid candidate revision")
+ config = json.loads((ROOT.parents[1] / "benchmarks/config.json").read_text())
+ client = SandboxClient(APIClient(api_key=os.environ["PRIME_SANDBOX_API_KEY"]))
+ sandbox = client.create(
+ CreateSandboxRequest(
+ name=f"behavioral-{side}-{run_id}-{attempt}",
+ docker_image=config["image"],
+ cpu_cores=config["cpu_cores"],
+ memory_gb=config["memory_gb"],
+ disk_size_gb=config["disk_gb"],
+ vm=False,
+ region=config["region"],
+ timeout_minutes=BUILD_SANDBOX_TIMEOUT_MINUTES,
+ team_id=os.environ.get("PRIME_TEAM_ID") or None,
+ labels=labels(repository, run_id, attempt, side),
+ idempotency_key=f"behavioral-{side}-{repository}-{run_id}-{attempt}",
+ )
+ )
+ try:
+ wait_running(client, sandbox.id)
+ setup = (
+ "set -eu; apt-get update -qq; "
+ "DEBIAN_FRONTEND=noninteractive apt-get install -y -qq git util-linux; "
+ f"mkdir -p {REMOTE}; useradd --create-home --uid 1500 --shell /bin/bash builder"
+ )
+ result = client.execute_command(sandbox.id, setup, timeout=SETUP_TIMEOUT_SECONDS)
+ if result.exit_code:
+ raise RuntimeError("candidate build sandbox setup failed")
+ client.upload_file(sandbox.id, f"{REMOTE}/builder.py", str(ROOT / "builder.py"))
+ command = shlex.join(
+ [
+ "python3",
+ f"{REMOTE}/builder.py",
+ "--repository",
+ source_repository,
+ "--sha",
+ sha,
+ ]
+ )
+ exit_code = run_builder(client, sandbox.id, command)
+ client.execute_command(
+ sandbox.id,
+ "pkill -KILL -u builder || true",
+ timeout=CONTROL_COMMAND_TIMEOUT_SECONDS,
+ )
+ output.mkdir(parents=True, exist_ok=True)
+ tail = client.execute_command(
+ sandbox.id,
+ f"(tail -c 900000 -- {REMOTE}/results/build.log 2>/dev/null || true; "
+ f"tail -c 100000 -- {REMOTE}/results/controller.log 2>/dev/null || true) "
+ f"> {REMOTE}/results/build-tail.log",
+ timeout=CONTROL_COMMAND_TIMEOUT_SECONDS,
+ )
+ if tail.exit_code == 0:
+ client.download_file(
+ sandbox.id,
+ f"{REMOTE}/results/build-tail.log",
+ str(output / "build.log"),
+ )
+ if exit_code:
+ raise RuntimeError("candidate release build failed; see the build log artifact")
+ client.download_file(
+ sandbox.id,
+ f"{REMOTE}/results/artifact-manifest.json",
+ str(output / "artifact-manifest.json"),
+ )
+ manifest_path = output / "artifact-manifest.json"
+ if manifest_path.stat().st_size > 100_000:
+ raise ValueError("candidate artifact manifest exceeds its size limit")
+ manifest = json.loads(manifest_path.read_text())
+ records = manifest.get("artifacts") if manifest.get("sha") == sha else None
+ if (
+ not isinstance(records, list)
+ or len(records) != 4
+ or {record.get("name") for record in records} != EXPECTED
+ ):
+ raise ValueError("candidate artifact manifest is incomplete")
+ source = f"{REMOTE}/results/artifacts"
+ for record in records:
+ name = record["name"]
+ size = record.get("size")
+ digest = record.get("sha256")
+ if not isinstance(size, int) or not 0 < size <= MAX_ARTIFACT_BYTES:
+ raise ValueError("candidate artifact size is invalid")
+ if not isinstance(digest, str) or not re.fullmatch(r"[0-9a-f]{64}", digest):
+ raise ValueError("candidate artifact digest is invalid")
+ target = output / name
+ client.download_file(sandbox.id, f"{source}/{name}", str(target))
+ if target.stat().st_size != size:
+ raise ValueError("candidate artifact size changed after the build")
+ if hashlib.sha256(target.read_bytes()).hexdigest() != digest:
+ raise ValueError("candidate artifact digest changed after the build")
+ finally:
+ client.delete(sandbox.id)
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--repository", required=True)
+ parser.add_argument("--source-repository", required=True)
+ parser.add_argument("--sha", required=True)
+ parser.add_argument("--run-id", required=True, type=int)
+ parser.add_argument("--attempt", required=True, type=int)
+ parser.add_argument("--output", required=True, type=Path)
+ parser.add_argument("--side", required=True, choices=("base", "head"))
+ args = parser.parse_args()
+ build(
+ args.repository,
+ args.source_repository,
+ args.sha,
+ args.run_id,
+ args.attempt,
+ args.output,
+ args.side,
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/evals/short_swe/builder.py b/scripts/evals/short_swe/builder.py
new file mode 100644
index 0000000000..a5def9f0b2
--- /dev/null
+++ b/scripts/evals/short_swe/builder.py
@@ -0,0 +1,207 @@
+#!/usr/bin/env python3
+"""Build candidate release tarballs inside an untrusted Prime sandbox."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import pwd
+import re
+import stat
+import subprocess
+from pathlib import Path
+
+VERSION = "0.0.0-benchmark"
+ROOT = Path("/opt/behavioral-build")
+SOURCE = ROOT / "source"
+RESULTS = ROOT / "results"
+SNAPSHOT = RESULTS / "artifacts"
+ARTIFACT_RELATIVE = Path("packages/coding-agent/release/behavioral/artifacts")
+MAX_ARTIFACT_BYTES = 20_000_000
+SHA_RE = re.compile(r"^[0-9a-f]{40}$")
+REPO_RE = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$")
+
+
+def run_as_builder(args: list[str], cwd: Path, timeout: int = 600) -> None:
+ env = {
+ **os.environ,
+ "HOME": "/home/builder",
+ "CI": "1",
+ "npm_config_audit": "false",
+ "npm_config_fund": "false",
+ }
+ with (RESULTS / "build.log").open("a") as log:
+ result = subprocess.run(
+ ["runuser", "-u", "builder", "--", *args],
+ cwd=cwd,
+ env=env,
+ stdout=log,
+ stderr=subprocess.STDOUT,
+ text=True,
+ timeout=timeout,
+ check=False,
+ )
+ if result.returncode:
+ raise RuntimeError(f"candidate build command failed with exit {result.returncode}")
+
+
+def open_candidate_artifacts() -> int:
+ flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC
+ directory = os.open(SOURCE, flags)
+ try:
+ for part in ARTIFACT_RELATIVE.parts:
+ child = os.open(part, flags, dir_fd=directory)
+ os.close(directory)
+ directory = child
+ return directory
+ except BaseException:
+ os.close(directory)
+ raise
+
+
+def create_snapshot(path: Path = SNAPSHOT) -> int:
+ path.mkdir(mode=0o700)
+ metadata = path.stat(follow_symlinks=False)
+ if not stat.S_ISDIR(metadata.st_mode) or stat.S_IMODE(metadata.st_mode) != 0o700:
+ raise ValueError("trusted artifact snapshot permissions are invalid")
+ return os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC)
+
+
+def snapshot_artifact(source: int, destination: int, name: str) -> dict[str, int | str]:
+ try:
+ metadata = os.stat(name, dir_fd=source, follow_symlinks=False)
+ if not stat.S_ISREG(metadata.st_mode) or not 0 < metadata.st_size <= MAX_ARTIFACT_BYTES:
+ raise ValueError("candidate artifact is not a bounded regular file")
+ source_file = os.open(
+ name,
+ os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC,
+ dir_fd=source,
+ )
+ except OSError as error:
+ raise ValueError("candidate artifact could not be opened safely") from error
+
+ destination_file = None
+ try:
+ opened = os.fstat(source_file)
+ if (
+ not stat.S_ISREG(opened.st_mode)
+ or opened.st_size != metadata.st_size
+ or (opened.st_dev, opened.st_ino) != (metadata.st_dev, metadata.st_ino)
+ ):
+ raise ValueError("candidate artifact changed before its safe open")
+ destination_file = os.open(
+ name,
+ os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_CLOEXEC,
+ 0o600,
+ dir_fd=destination,
+ )
+ digest = hashlib.sha256()
+ remaining = opened.st_size
+ while remaining:
+ chunk = os.read(source_file, min(1_000_000, remaining))
+ if not chunk:
+ raise ValueError("candidate artifact changed while being copied")
+ digest.update(chunk)
+ view = memoryview(chunk)
+ while view:
+ written = os.write(destination_file, view)
+ if written <= 0:
+ raise OSError("candidate artifact snapshot write failed")
+ view = view[written:]
+ remaining -= len(chunk)
+ if os.read(source_file, 1) or os.fstat(source_file).st_size != opened.st_size:
+ raise ValueError("candidate artifact changed while being copied")
+ os.fsync(destination_file)
+ return {"name": name, "size": opened.st_size, "sha256": digest.hexdigest()}
+ except BaseException:
+ if destination_file is not None:
+ try:
+ os.unlink(name, dir_fd=destination)
+ except FileNotFoundError:
+ pass
+ raise
+ finally:
+ os.close(source_file)
+ if destination_file is not None:
+ os.close(destination_file)
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--repository", required=True)
+ parser.add_argument("--sha", required=True)
+ args = parser.parse_args()
+ if not REPO_RE.fullmatch(args.repository) or not SHA_RE.fullmatch(args.sha):
+ raise ValueError("invalid source repository or revision")
+
+ RESULTS.mkdir(parents=True, exist_ok=True)
+ SOURCE.mkdir(parents=True, exist_ok=True)
+ builder = pwd.getpwnam("builder")
+ os.chown(SOURCE, builder.pw_uid, builder.pw_gid)
+ source_url = f"https://github.com/{args.repository}.git"
+ for command in (
+ ["git", "init", "--quiet"],
+ ["git", "fetch", "--depth=1", source_url, args.sha],
+ ["git", "checkout", "--detach", "FETCH_HEAD"],
+ ["npm", "ci", "--no-audit", "--no-fund"],
+ ):
+ run_as_builder(command, SOURCE)
+ actual = subprocess.run(
+ ["runuser", "-u", "builder", "--", "git", "rev-parse", "HEAD"],
+ cwd=SOURCE,
+ check=True,
+ capture_output=True,
+ text=True,
+ ).stdout.strip()
+ if actual != args.sha:
+ raise RuntimeError("candidate checkout resolved to another revision")
+
+ for package in ("tui", "ai", "agent", "coding-agent"):
+ run_as_builder(
+ [str(SOURCE / "node_modules/.bin/tsgo"), "-p", "tsconfig.build.json"],
+ SOURCE / "packages" / package,
+ timeout=240,
+ )
+ agent = SOURCE / "packages/coding-agent"
+ run_as_builder(["chmod", "+x", "dist/cli.js"], agent)
+ for script in ("copy-assets", "bundle"):
+ run_as_builder(["npm", "run", script], agent, timeout=240)
+ run_as_builder(
+ [
+ "node",
+ "scripts/pack-prime-agent-release.mjs",
+ "--base-url",
+ "https://invalid.local/releases",
+ "--version",
+ VERSION,
+ "--out-dir",
+ "packages/coding-agent/release/behavioral",
+ ],
+ SOURCE,
+ timeout=240,
+ )
+ expected = {
+ f"prime-agent-{VERSION}.tgz",
+ f"prime-agent-ai-{VERSION}.tgz",
+ f"prime-agent-core-{VERSION}.tgz",
+ f"prime-agent-tui-{VERSION}.tgz",
+ }
+ source = open_candidate_artifacts()
+ destination = create_snapshot()
+ try:
+ found = {name for name in os.listdir(source) if name.endswith(".tgz")}
+ if found != expected:
+ raise RuntimeError("candidate build did not produce the expected release tarballs")
+ records = [snapshot_artifact(source, destination, name) for name in sorted(expected)]
+ finally:
+ os.close(source)
+ os.close(destination)
+ (RESULTS / "artifact-manifest.json").write_text(
+ json.dumps({"sha": args.sha, "artifacts": records}, indent=2) + "\n"
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/evals/short_swe/ci.py b/scripts/evals/short_swe/ci.py
new file mode 100644
index 0000000000..10c54ba2dc
--- /dev/null
+++ b/scripts/evals/short_swe/ci.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+"""Resolve one exact label-approved PR comparison from a trusted workflow."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+from pathlib import Path
+
+LABEL = "pre-release"
+SHA_RE = re.compile(r"^[0-9a-f]{40}$")
+REPO_RE = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$")
+
+
+def resolve(output: Path, harness_sha: str) -> dict:
+ event = json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text())
+ pull = event.get("pull_request")
+ if (
+ event.get("action") != "labeled"
+ or (event.get("label") or {}).get("name") != LABEL
+ or not isinstance(pull, dict)
+ ):
+ raise ValueError("behavioral evaluation requires the pre-release label event")
+ repository = os.environ["GITHUB_REPOSITORY"]
+ head_repository = (pull.get("head", {}).get("repo") or {}).get("full_name", "")
+ values = {
+ "schema_version": 1,
+ "repository": repository,
+ "head_repository": head_repository,
+ "pr": pull.get("number"),
+ "run_id": int(os.environ["GITHUB_RUN_ID"]),
+ "attempt": int(os.environ["GITHUB_RUN_ATTEMPT"]),
+ "harness_sha": harness_sha,
+ "base_sha": pull.get("base", {}).get("sha"),
+ "head_sha": pull.get("head", {}).get("sha"),
+ }
+ if (
+ pull.get("state") != "open"
+ or pull.get("base", {}).get("repo", {}).get("full_name") != repository
+ or not REPO_RE.fullmatch(repository)
+ or not REPO_RE.fullmatch(head_repository)
+ or not isinstance(values["pr"], int)
+ or values["pr"] <= 0
+ or any(not SHA_RE.fullmatch(str(values[key])) for key in ("harness_sha", "base_sha", "head_sha"))
+ or values["harness_sha"] != values["base_sha"]
+ ):
+ raise ValueError("invalid pull request identity")
+ output.mkdir(parents=True, exist_ok=True)
+ (output / "request.json").write_text(json.dumps(values, indent=2, sort_keys=True) + "\n")
+ github_output = os.environ.get("GITHUB_OUTPUT")
+ if github_output:
+ with Path(github_output).open("a") as stream:
+ for key in ("pr", "head_repository", "base_sha", "head_sha"):
+ stream.write(f"{key}={values[key]}\n")
+ return values
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--output", type=Path, default=Path("request"))
+ parser.add_argument("--harness-sha", required=True)
+ args = parser.parse_args()
+ resolve(args.output, args.harness_sha)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/evals/short_swe/cleanup.py b/scripts/evals/short_swe/cleanup.py
new file mode 100644
index 0000000000..2e63ed89ff
--- /dev/null
+++ b/scripts/evals/short_swe/cleanup.py
@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+"""Delete Prime sandboxes owned by one behavioral-evaluation generation."""
+
+from __future__ import annotations
+
+import argparse
+import os
+import re
+
+from prime_sandboxes import APIClient, SandboxClient
+
+REPO_RE = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$")
+
+
+def cleanup_owned(client, labels: list[str], team_id: str | None = None) -> int:
+ sandbox_ids = []
+ page = 1
+ while True:
+ response = client.list(
+ team_id=team_id,
+ labels=labels,
+ page=page,
+ per_page=100,
+ exclude_terminated=True,
+ )
+ sandbox_ids.extend(sandbox.id for sandbox in response.sandboxes if set(labels) <= set(sandbox.labels))
+ if not response.has_next:
+ break
+ page += 1
+ failures = []
+ for sandbox_id in sandbox_ids:
+ try:
+ client.delete(sandbox_id)
+ except Exception:
+ failures.append(sandbox_id)
+ if failures:
+ raise RuntimeError(f"cleanup failed for {len(failures)} sandbox(es); TTL remains active")
+ return len(sandbox_ids)
+
+
+def cleanup(repository: str, run_id: int, attempt: int) -> int:
+ if not REPO_RE.fullmatch(repository):
+ raise ValueError("invalid repository")
+ labels = [
+ "prime-agent-behavioral-v1",
+ f"repository:{repository}",
+ f"run:{run_id}",
+ f"attempt:{attempt}",
+ ]
+ client = SandboxClient(APIClient(api_key=os.environ["PRIME_SANDBOX_API_KEY"]))
+ return cleanup_owned(client, labels, os.environ.get("PRIME_TEAM_ID") or None)
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--repository", required=True)
+ parser.add_argument("--run-id", required=True, type=int)
+ parser.add_argument("--attempt", required=True, type=int)
+ args = parser.parse_args()
+ count = cleanup(args.repository, args.run_id, args.attempt)
+ print(f"Deleted {count} behavioral-evaluation sandboxes.")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/evals/short_swe/evaluate.py b/scripts/evals/short_swe/evaluate.py
new file mode 100644
index 0000000000..07556c088a
--- /dev/null
+++ b/scripts/evals/short_swe/evaluate.py
@@ -0,0 +1,295 @@
+#!/usr/bin/env python3
+"""Run the fixed base/head Short SWE comparison and read native Verifiers episodes."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import os
+import subprocess
+import time
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent
+TRACE_FILE_LIMIT = 50_000_000
+
+
+def task_name(trace) -> str:
+ name = getattr(trace.task.data, "name", None)
+ if not isinstance(name, str) or not name:
+ raise ValueError("trace has no task name")
+ return name.rsplit("/", 1)[-1]
+
+
+def scored(trace) -> bool:
+ return bool(trace.rewards) and all(reward is not None for reward in trace.rewards.values())
+
+
+def recognized_model_failure(trace) -> bool:
+ if len(trace.errors) != 1:
+ return False
+ terminal = trace.errors[0]
+ call_errors = [call.error for call in trace.calls if call.error is not None]
+ if terminal.type == "ProviderError" and terminal.status_code in {400, 413, 422}:
+ return bool(call_errors) and all(
+ error.type == terminal.type
+ and error.status_code == terminal.status_code
+ and error.message == terminal.message
+ for error in call_errors
+ )
+ if terminal.type == "ProviderError" and terminal.status_code in {500, 502, 503, 504}:
+ # Retryable call errors are recorded on the exchanges the client retried, so a rate
+ # limit under the shared concurrency limit does not contradict a terminal outage.
+ transient = {429, 500, 502, 503, 504}
+ provider_call_errors = [error for error in call_errors if error.type == "ProviderError"]
+ return all(error.status_code in transient for error in provider_call_errors)
+ return (
+ not call_errors
+ and terminal.type == "HarnessError"
+ and terminal.message.startswith("agent timeout: rollout exceeded its ")
+ and terminal.message.endswith(" budget")
+ )
+
+
+def accepted_model_outcome(trace) -> bool:
+ if not trace.is_completed:
+ return False
+ if trace.ok:
+ return scored(trace)
+ return recognized_model_failure(trace) and not trace.rewards
+
+
+def validate_graph(trace) -> None:
+ for index, node in enumerate(trace.nodes):
+ parent = node.parent
+ if parent is not None and (
+ not isinstance(parent, int) or isinstance(parent, bool) or not 0 <= parent < index
+ ):
+ raise ValueError("trace has an invalid parent link")
+ for call in trace.calls:
+ node = call.node
+ if node is None:
+ continue
+ if (
+ not isinstance(node, int)
+ or isinstance(node, bool)
+ or not 0 <= node < len(trace.nodes)
+ or not trace.nodes[node].sampled
+ ):
+ raise ValueError("model call has an invalid message node")
+
+
+def validate_timing(trace) -> float:
+ total = 0.0
+ for phase in ("boot", "setup", "agent", "finalize", "scoring"):
+ span = getattr(trace.timing, phase)
+ start, end = span.start, span.end
+ if (
+ any(
+ not isinstance(value, (int, float))
+ or isinstance(value, bool)
+ or not math.isfinite(value)
+ or value < 0
+ for value in (start, end)
+ )
+ or bool(start) != bool(end)
+ or end < start
+ ):
+ raise ValueError("trace has invalid timing")
+ total += span.duration
+ verifier = getattr(trace, "info", {}).get("isolated_verifier_seconds", 0.0)
+ if (
+ not isinstance(verifier, (int, float))
+ or isinstance(verifier, bool)
+ or not math.isfinite(verifier)
+ or verifier < 0
+ ):
+ raise ValueError("trace has invalid isolated verifier timing")
+ return total + verifier
+
+
+def provider_usage(trace, identity: str) -> tuple[int, int, int]:
+ """Provider token buckets, as zeroes when a model failure reported no usage."""
+ for call in trace.calls:
+ if call.error is not None:
+ continue
+ usage = call.usage
+ values = (
+ usage.prompt_tokens if usage else None,
+ usage.cached_input_tokens if usage else None,
+ usage.completion_tokens if usage else None,
+ )
+ if any(not isinstance(value, int) or isinstance(value, bool) or value < 0 for value in values):
+ raise ValueError(f"{identity} has incomplete provider usage")
+ usage = trace.usage
+ if usage is None:
+ if trace.ok:
+ raise ValueError(f"{identity} has incomplete provider usage")
+ return (0, 0, 0)
+ if usage.cached_input_tokens is None:
+ raise ValueError(f"{identity} has incomplete provider usage")
+ return (usage.prompt_tokens, usage.cached_input_tokens, usage.completion_tokens)
+
+
+def trace_record(episode, taskset: str) -> dict:
+ if len(episode.traces) != 1:
+ raise ValueError("single-agent Short SWE episode must contain one trace")
+ trace = episode.traces[0]
+ identity = f"{taskset}/{task_name(trace)}"
+ if episode.errors or not accepted_model_outcome(trace):
+ raise ValueError(f"{identity} did not produce a complete trace or model outcome")
+ validate_graph(trace)
+ _ = trace.branches
+ uncached_input_tokens, cached_input_tokens, output_tokens = provider_usage(trace, identity)
+ rewards = [
+ value for reward in trace.rewards.values() if reward for value in (reward.score, reward.weight)
+ ]
+ if any(isinstance(value, bool) or not math.isfinite(value) for value in rewards):
+ raise ValueError(f"{identity} has invalid reward values")
+ if not math.isfinite(trace.reward):
+ raise ValueError(f"{identity} has invalid aggregate reward")
+ models = {call.model for call in trace.calls if call.model is not None}
+ if len(models) != 1:
+ raise ValueError(f"{identity} did not use exactly one model")
+ return {
+ "taskset": taskset,
+ "task": task_name(trace),
+ "model": models.pop(),
+ "resolved": trace.ok and scored(trace) and trace.reward > 0,
+ "model_failure": not trace.ok,
+ "uncached_input_tokens": uncached_input_tokens,
+ "cached_input_tokens": cached_input_tokens,
+ "output_tokens": output_tokens,
+ "e2e_seconds": validate_timing(trace),
+ "model_calls": len(trace.calls),
+ }
+
+
+def read_taskset(path: Path, taskset: str) -> list[dict]:
+ from verifiers.v1.cli.output import read_episodes
+ from verifiers.v1.trace import WireTrace
+
+ files = list(path.rglob("traces.jsonl"))
+ if len(files) != 1:
+ raise ValueError(f"{taskset} produced {len(files)} trace files")
+ if files[0].stat().st_size > TRACE_FILE_LIMIT:
+ raise ValueError(f"{taskset} trace file exceeds its size limit")
+ return [trace_record(episode, taskset) for episode in read_episodes(files[0].parent, WireTrace)]
+
+
+def validate_tasks(records: list[dict], manifest: dict) -> None:
+ expected = {(item["id"], task) for item in manifest["tasksets"] for task in item["tasks"]}
+ actual = {(record["taskset"], record["task"]) for record in records}
+ if actual != expected or len(records) != len(expected):
+ raise ValueError("trace task identities differ from the fixed 15/8/5 manifest")
+ models = {record["model"] for record in records}
+ if len(models) != 1 or models.isdisjoint({manifest["model"], manifest["backup_model"]}):
+ raise ValueError("paired tasks must all use one pinned model")
+
+
+def validate_oracle_episode(episode) -> None:
+ if len(episode.traces) != 1:
+ raise RuntimeError("SWE-bench oracle did not produce one trace")
+ trace = episode.traces[0]
+ rewards = [
+ value for reward in trace.rewards.values() if reward for value in (reward.score, reward.weight)
+ ]
+ if any(isinstance(value, bool) or not math.isfinite(value) for value in rewards):
+ raise RuntimeError("SWE-bench gold-patch oracle produced non-finite rewards")
+ if (
+ episode.errors
+ or task_name(trace) != "astropy__astropy-14096"
+ or not trace.ok
+ or not trace.is_completed
+ or not scored(trace)
+ or not math.isfinite(trace.reward)
+ or trace.reward <= 0
+ ):
+ raise RuntimeError("SWE-bench gold-patch oracle did not resolve")
+
+
+def run_oracle(executable: Path, config: Path, output: Path) -> None:
+ from verifiers.v1.cli.output import read_episodes
+ from verifiers.v1.trace import WireTrace
+
+ target = output / "oracle"
+ target.mkdir(parents=True, exist_ok=True)
+ environment = {**os.environ, "PYTHONPATH": str(ROOT)}
+ with (target / "eval.log").open("w") as log:
+ completed = subprocess.run(
+ [str(executable), "@", str(config), "--output-dir", str(target), "--no-push"],
+ env=environment,
+ stdout=log,
+ stderr=subprocess.STDOUT,
+ check=False,
+ )
+ if completed.returncode:
+ raise RuntimeError(f"SWE-bench oracle exited with status {completed.returncode}")
+ files = list(target.rglob("traces.jsonl"))
+ if len(files) != 1 or files[0].stat().st_size > TRACE_FILE_LIMIT:
+ raise RuntimeError("SWE-bench oracle trace output is invalid")
+ episodes = list(read_episodes(files[0].parent, WireTrace))
+ if len(episodes) != 1:
+ raise RuntimeError("SWE-bench oracle did not produce one episode")
+ validate_oracle_episode(episodes[0])
+
+
+def read_existing(output: Path, configs: Path) -> dict[str, list[dict]]:
+ """Read pre-collected episode files (hosted evaluations pulled by `hosted_eval.py`)."""
+ return {
+ side: [
+ record
+ for taskset in sorted((configs / side).glob("*.toml"))
+ for record in read_taskset(output / side / taskset.stem, taskset.stem)
+ ]
+ for side in ("base", "head")
+ }
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--eval", required=True, type=Path)
+ parser.add_argument("--configs", required=True, type=Path)
+ parser.add_argument("--output", required=True, type=Path)
+ parser.add_argument("--request", required=True, type=Path)
+ parser.add_argument("--result", required=True, type=Path)
+ mode = parser.add_mutually_exclusive_group()
+ mode.add_argument("--oracle-only", action="store_true")
+ mode.add_argument("--from-existing", action="store_true")
+ args = parser.parse_args()
+ manifest = json.loads((ROOT / "short-swe.json").read_text())
+ request = json.loads(args.request.read_text())
+ try:
+ if args.oracle_only:
+ run_oracle(args.eval, args.configs / "oracle.toml", args.output)
+ return
+ started = time.time()
+ if args.from_existing:
+ sides = read_existing(args.output, args.configs)
+ else:
+ run_oracle(args.eval, args.configs / "oracle.toml", args.output)
+ sides = read_existing(args.output, args.configs)
+ for records in sides.values():
+ validate_tasks(records, manifest)
+ models = {record["model"] for records in sides.values() for record in records}
+ if len(models) != 1:
+ raise ValueError("base and head did not use the same model")
+ result = {
+ "schema_version": 1,
+ "request": request,
+ "model": models.pop(),
+ "started_at": started,
+ "finished_at": time.time(),
+ "sides": sides,
+ }
+ args.result.parent.mkdir(parents=True, exist_ok=True)
+ args.result.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n")
+ except Exception as error:
+ args.output.mkdir(parents=True, exist_ok=True)
+ (args.output / "failure.txt").write_text(f"{type(error).__name__}: {error}\n")
+ raise
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/evals/short_swe/hosted_eval.py b/scripts/evals/short_swe/hosted_eval.py
new file mode 100644
index 0000000000..cff8f1d548
--- /dev/null
+++ b/scripts/evals/short_swe/hosted_eval.py
@@ -0,0 +1,302 @@
+"""Launch hosted Prime Evals runs for the Short SWE suite and collect their episodes.
+
+The workflow builds the candidate tarballs, uploads them as GitHub Actions
+artifacts, and uses this script to launch one hosted evaluation per configured side and
+taskset with the artifact delivered through `custom_secrets`. Each hosted run
+executes the private `short-swe-*` Environments Hub packages; their episodes are
+pulled back into local `traces.jsonl` files and validated by the same
+`evaluate.read_taskset` gates used by the local paired evaluation.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent
+sys.path.insert(0, str(ROOT.parent.parent))
+
+GITHUB_JOB_LIMIT_MINUTES = 360
+"""GitHub kills a hosted job at this limit, so every budget below has to fit inside it."""
+
+HOSTED_EVALUATION_TIMEOUT_MINUTES = 240
+"""Per-evaluation platform timeout, capped further by the job budget left at launch."""
+
+COLLECT_RESERVE_MINUTES = 20
+"""Time kept back after the wait for collection, pairing, and reporting."""
+
+POLL_TIMEOUT_SECONDS = 120
+"""Bound on one `prime eval get` call, so a stalled request cannot pass the wait deadline."""
+
+COLLECT_TIMEOUT_SECONDS = 600
+"""Bound on one `prime eval samples` call, inside the collection reserve."""
+
+STOP_TIMEOUT_SECONDS = 60
+"""Bound on one `prime eval stop` call, so six stalls still leave the finisher time to run."""
+
+JOB_DEADLINE_ENV = "BEHAVIORAL_JOB_DEADLINE_EPOCH"
+"""The workflow exports the epoch when the runner kills this job."""
+
+HOSTED_ENVIRONMENTS = {
+ "swebench-verified": "primeintellect/[email protected]",
+ "swebench-pro": "primeintellect/[email protected]",
+ "scaleswe": "primeintellect/[email protected]",
+}
+EVAL_ID_RE = re.compile(r"Evaluation ID: (\S+)")
+TERMINAL_STATUSES = {"COMPLETED", "FAILED", "TIMEOUT", "CANCELLED"}
+
+
+def run(command: list[str], *, env: dict[str, str] | None = None, timeout: float | None = None) -> str:
+ try:
+ result = subprocess.run(
+ command,
+ capture_output=True,
+ text=True,
+ env={**os.environ, **(env or {})},
+ timeout=timeout,
+ )
+ except subprocess.TimeoutExpired as error:
+ raise RuntimeError(f"{command[0]} did not answer within {timeout:.0f}s") from error
+ if result.returncode:
+ raise RuntimeError(f"{command[0]} failed: {result.stdout}\n{result.stderr}")
+ return result.stdout
+
+
+def job_seconds_left() -> float:
+ """Seconds until GitHub kills this job, taken from the deadline the workflow exports."""
+ raw = os.environ.get(JOB_DEADLINE_ENV)
+ if raw is None:
+ raise RuntimeError(f"{JOB_DEADLINE_ENV} is not set, so the job budget is unknown")
+ try:
+ return float(raw) - time.time()
+ except ValueError as error:
+ raise RuntimeError(f"{JOB_DEADLINE_ENV} is not a timestamp") from error
+
+
+def wait_deadline() -> float:
+ """Polling stops before the runner would kill the job and skip the stop path."""
+ return time.time() + job_seconds_left() - COLLECT_RESERVE_MINUTES * 60
+
+
+def evaluation_timeout_minutes() -> int:
+ """Cap a hosted run so its platform timeout and the collection both fit the job budget."""
+ remaining = job_seconds_left() - COLLECT_RESERVE_MINUTES * 60
+ if remaining <= 0:
+ raise RuntimeError("the behavioral job budget is exhausted before launch")
+ return max(1, min(HOSTED_EVALUATION_TIMEOUT_MINUTES, int(remaining // 60)))
+
+
+def launch(args: argparse.Namespace) -> None:
+ request = json.loads(Path(args.request).read_text())
+ manifest = json.loads((ROOT / "short-swe.json").read_text())
+ model = args.model or manifest["model"]
+ if model not in {manifest["model"], manifest["backup_model"]}:
+ raise ValueError(f"model {model!r} is not pinned by the Short SWE manifest")
+ sizes = {item["id"]: len(item["tasks"]) for item in manifest["tasksets"]}
+ sources = json.loads(Path(args.sources).read_text())
+ runs: dict[str, dict] = {}
+ logs = Path(args.output) / "launch-logs"
+ logs.mkdir(parents=True, exist_ok=True)
+ sides = tuple(sources)
+ for side in sides:
+ for taskset_id, environment in HOSTED_ENVIRONMENTS.items():
+ source = sources[side]
+ secrets = {
+ "CANDIDATE_TARBALLS_URL": source["tarballs_url"],
+ "CANDIDATE_COMMIT": source["commit"],
+ "CANDIDATE_CHECKSUMS": json.dumps(source["checksums"]),
+ "CANDIDATE_TOKEN": source.get("token", ""),
+ }
+ name = f"{request['pr']}-{side}-{taskset_id}-{request['head_sha'][:9]}"
+ command = [
+ "prime",
+ "eval",
+ "run",
+ environment,
+ "--hosted",
+ "-m",
+ model,
+ "-n",
+ str(sizes[taskset_id]),
+ "-r",
+ str(manifest["num_rollouts"]),
+ "--max-concurrent",
+ str(manifest["max_concurrent"]),
+ "--timeout-minutes",
+ str(evaluation_timeout_minutes()),
+ "--eval-name",
+ name,
+ "--custom-secrets",
+ json.dumps(secrets),
+ ]
+ log_path = logs / f"{side}-{taskset_id}.log"
+ log = log_path.open("w")
+ try:
+ subprocess.Popen(
+ command,
+ stdout=log,
+ stderr=subprocess.STDOUT,
+ text=True,
+ )
+ except BaseException:
+ log.close()
+ stop_started(runs)
+ raise
+ runs[f"{side}/{taskset_id}"] = {
+ "environment": environment,
+ "name": name,
+ "log": str(log_path),
+ "num_examples": sizes[taskset_id],
+ }
+ (Path(args.output) / "hosted-runs.json").write_text(json.dumps(runs, indent=2))
+ # Each launch exits right after the platform assigns an evaluation id.
+ deadline = time.time() + 300
+ assigned: dict[str, str] = {}
+ while len(assigned) < len(runs) and time.time() < deadline:
+ for key, record in runs.items():
+ if key in assigned:
+ continue
+ text = Path(record["log"]).read_text(errors="replace")
+ match = EVAL_ID_RE.search(text)
+ if match:
+ assigned[key] = match.group(1)
+ time.sleep(5)
+ missing = [key for key in runs if key not in assigned]
+ if missing:
+ stop_started(runs)
+ for key in missing:
+ print(Path(runs[key]["log"]).read_text(errors="replace")[-2000:], file=sys.stderr)
+ raise RuntimeError(f"hosted evaluations did not start: {missing}")
+ for key, evaluation_id in assigned.items():
+ runs[key]["evaluation_id"] = evaluation_id
+ (Path(args.output) / "hosted-runs.json").write_text(json.dumps(runs, indent=2))
+ print(json.dumps({key: value["evaluation_id"] for key, value in runs.items()}, indent=2))
+
+
+def stop_started(runs: dict[str, dict]) -> None:
+ """Stop every hosted evaluation a partial launch already created."""
+ for record in runs.values():
+ evaluation_id = record.get("evaluation_id")
+ if not evaluation_id:
+ try:
+ text = Path(record["log"]).read_text(errors="replace")
+ except OSError:
+ continue
+ match = EVAL_ID_RE.search(text)
+ if not match:
+ continue
+ evaluation_id = match.group(1)
+ try:
+ subprocess.run(
+ ["prime", "eval", "stop", evaluation_id],
+ capture_output=True,
+ text=True,
+ timeout=STOP_TIMEOUT_SECONDS,
+ )
+ except subprocess.TimeoutExpired:
+ print(f"{evaluation_id}: stop did not answer, leaving it to its own deadline", file=sys.stderr)
+
+
+def stop(args: argparse.Namespace) -> None:
+ """Stop hosted evaluations a cancelled or failed job left running."""
+ try:
+ runs = json.loads((Path(args.output) / "hosted-runs.json").read_text())
+ except (OSError, json.JSONDecodeError):
+ return
+ stop_started(runs)
+
+
+def wait(args: argparse.Namespace) -> None:
+ runs = json.loads((Path(args.output) / "hosted-runs.json").read_text())
+ deadline = wait_deadline()
+ failures = []
+ try:
+ for key, record in runs.items():
+ while True:
+ remaining = deadline - time.time()
+ if remaining <= 0:
+ raise RuntimeError(f"{key}: hosted evaluation did not finish inside the job budget")
+ detail = json.loads(
+ run(
+ ["prime", "eval", "get", record["evaluation_id"], "--output", "json"],
+ timeout=min(POLL_TIMEOUT_SECONDS, remaining),
+ )
+ )
+ status = detail.get("status") or detail.get("evaluation", {}).get("status")
+ if status in TERMINAL_STATUSES:
+ break
+ time.sleep(30)
+ if status != "COMPLETED":
+ failures.append(f"{key}: {status} {detail.get('error_message', '')}")
+ except BaseException:
+ stop_started(runs)
+ raise
+ if failures:
+ raise RuntimeError("; ".join(failures))
+
+
+def collect(args: argparse.Namespace) -> None:
+ runs = json.loads((Path(args.output) / "hosted-runs.json").read_text())
+ deadline = time.time() + COLLECT_RESERVE_MINUTES * 60
+ for key, record in runs.items():
+ side, taskset_id = key.split("/", 1)
+ target = Path(args.output) / "raw-eval" / side / taskset_id
+ target.mkdir(parents=True, exist_ok=True)
+ remaining = deadline - time.time()
+ if remaining <= 0:
+ raise RuntimeError("collection exceeded its reserve before writing every episode")
+ payload = json.loads(
+ run(
+ ["prime", "eval", "samples", record["evaluation_id"], "--output", "json"],
+ timeout=min(COLLECT_TIMEOUT_SECONDS, remaining),
+ )
+ )
+ samples = payload.get("samples") or []
+ expected = record.get("num_examples")
+ if expected is not None and len(samples) != expected:
+ raise RuntimeError(f"{key}: expected {expected} samples, found {len(samples)}")
+ episodes = []
+ for sample in samples:
+ info = sample.get("info") or {}
+ native = info.get("native_wrapper")
+ if not native:
+ raise RuntimeError(f"{key}: sample without a native episode record")
+ episodes.append(json.dumps(native))
+ (target / "traces.jsonl").write_text("\n".join(episodes) + "\n")
+ print(f"{key}: collected {len(episodes)} episodes -> {target}")
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser()
+ sub = parser.add_subparsers(dest="command", required=True)
+ launch_parser = sub.add_parser("launch")
+ launch_parser.add_argument("--request", required=True)
+ launch_parser.add_argument("--sources", required=True)
+ launch_parser.add_argument("--output", required=True)
+ launch_parser.add_argument(
+ "--model",
+ default=None,
+ help="Override the pinned model with its backup when the primary is saturated.",
+ )
+ launch_parser.set_defaults(func=launch)
+ wait_parser = sub.add_parser("wait")
+ wait_parser.add_argument("--output", required=True)
+ wait_parser.set_defaults(func=wait)
+ collect_parser = sub.add_parser("collect")
+ collect_parser.add_argument("--output", required=True)
+ collect_parser.set_defaults(func=collect)
+ stop_parser = sub.add_parser("stop")
+ stop_parser.add_argument("--output", required=True)
+ stop_parser.set_defaults(func=stop)
+ args = parser.parse_args()
+ args.func(args)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/evals/short_swe/offline_swebench_grader.py b/scripts/evals/short_swe/offline_swebench_grader.py
new file mode 100644
index 0000000000..67914b3134
--- /dev/null
+++ b/scripts/evals/short_swe/offline_swebench_grader.py
@@ -0,0 +1,232 @@
+"""Dependency-free SWE-bench Verified log parser for the pinned task slice.
+
+Parser behavior is adapted from SWE-bench 4.0.3. Missing expected statuses fail as
+infrastructure; skipped expected tests remain unresolved.
+
+MIT License
+
+Copyright (c) 2023 Carlos E Jimenez, John Yang, Alexander Wettig, Shunyu Yao,
+Kexin Pei, Ofir Press, Karthik R Narasimhan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+
+STATUSES = {"PASSED", "FAILED", "SKIPPED", "ERROR", "XFAIL"}
+NONPASSING = {"FAILED", "SKIPPED", "ERROR"}
+
+
+class StatusMap(dict[str, str]):
+ """Keep any observed non-passing status from being overwritten by later output."""
+
+ def __setitem__(self, name: str, value: str) -> None:
+ current = self.get(name)
+ if current in NONPASSING:
+ return
+ if current is None or value in NONPASSING:
+ super().__setitem__(name, value)
+
+
+def pytest_log(log: str, _config: dict) -> dict[str, str]:
+ status = StatusMap()
+ for line in log.splitlines():
+ if any(line.startswith(value) for value in STATUSES):
+ if line.startswith("FAILED"):
+ line = line.replace(" - ", " ")
+ fields = line.split()
+ if len(fields) > 1:
+ status[fields[1]] = fields[0]
+ return status
+
+
+def pytest_options_log(log: str, config: dict) -> dict[str, str]:
+ status = StatusMap()
+ option_pattern = re.compile(r"(.*?)\[(.*)\]")
+ for line in log.splitlines():
+ if not any(line.startswith(value) for value in STATUSES):
+ continue
+ if line.startswith("FAILED"):
+ line = line.replace(" - ", " ")
+ fields = line.split()
+ if len(fields) <= 1:
+ continue
+ match = option_pattern.search(fields[1])
+ if match:
+ main, option = match.groups()
+ if option.startswith("/") and not option.startswith("//") and "*" not in option:
+ option = "/" + option.split("/")[-1]
+ name = f"{main}[{option}]"
+ else:
+ name = fields[1]
+ status[name] = fields[0]
+ return status
+
+
+def pytest_v2_log(log: str, _config: dict) -> dict[str, str]:
+ status = StatusMap()
+ controls = "".join(chr(value) for value in range(1, 32))
+ translator = str.maketrans("", "", controls)
+ for line in log.splitlines():
+ line = re.sub(r"\[(\d+)m", "", line).translate(translator)
+ if any(line.startswith(value) for value in STATUSES):
+ if line.startswith("FAILED"):
+ line = line.replace(" - ", " ")
+ fields = line.split()
+ if len(fields) >= 2:
+ status[fields[1]] = fields[0]
+ elif any(line.endswith(value) for value in STATUSES):
+ fields = line.split()
+ if len(fields) >= 2:
+ status[fields[0]] = fields[1]
+ return status
+
+
+def django_log(log: str, _config: dict) -> dict[str, str]:
+ status = StatusMap()
+ previous = None
+ for line in log.splitlines():
+ line = line.strip()
+ if "--version is equivalent to version" in line:
+ status["--version is equivalent to version"] = "PASSED"
+ if " ... " in line:
+ previous = line.split(" ... ")[0]
+ for suffix in (" ... ok", " ... OK", " ... OK"):
+ if line.endswith(suffix):
+ if line.startswith("Applying sites.0002_alter_domain_unique...test_no_migrations"):
+ line = line.split("...", 1)[-1].strip()
+ status[line.rsplit(suffix, 1)[0]] = "PASSED"
+ break
+ if " ... skipped" in line:
+ status[line.split(" ... skipped")[0]] = "SKIPPED"
+ if line.endswith(" ... FAIL"):
+ status[line.split(" ... FAIL")[0]] = "FAILED"
+ if line.startswith("FAIL:"):
+ status[line.split()[1].strip()] = "FAILED"
+ if line.endswith(" ... ERROR"):
+ status[line.split(" ... ERROR")[0]] = "ERROR"
+ if line.startswith("ERROR:"):
+ status[line.split()[1].strip()] = "ERROR"
+ if line.lstrip().startswith("ok") and previous is not None:
+ status[previous] = "PASSED"
+ patterns = (
+ r"^(.*?)\s\.\.\.\sTesting\ against\ Django\ installed\ in\ ((?s:.*?))\ silenced\)\.\nok$",
+ r"^(.*?)\s\.\.\.\sInternal\ Server\ Error:\ \/(.*)\/\nok$",
+ r"^(.*?)\s\.\.\.\sSystem check identified no issues \(0 silenced\)\nok$",
+ )
+ for pattern in patterns:
+ for match in re.finditer(pattern, log, re.MULTILINE):
+ status[match.group(1)] = "PASSED"
+ return status
+
+
+def seaborn_log(log: str, _config: dict) -> dict[str, str]:
+ status = StatusMap()
+ for line in log.splitlines():
+ fields = line.split()
+ if line.startswith("FAILED"):
+ if len(fields) > 1:
+ status[fields[1]] = "FAILED"
+ elif " PASSED " in line:
+ if len(fields) > 1 and fields[1] == "PASSED":
+ status[fields[0]] = "PASSED"
+ elif line.startswith("PASSED"):
+ if len(fields) > 1:
+ status[fields[1]] = "PASSED"
+ return status
+
+
+def sympy_log(log: str, _config: dict) -> dict[str, str]:
+ status = StatusMap()
+ for match in re.findall(r"(_*) (.*)\.py:(.*) (_*)", log):
+ status[f"{match[1]}.py:{match[2]}"] = "FAILED"
+ for line in log.splitlines():
+ line = line.strip()
+ if not line.startswith("test_"):
+ continue
+ if line.endswith(" E"):
+ status[line.split()[0]] = "ERROR"
+ if line.endswith(" F"):
+ status[line.split()[0]] = "FAILED"
+ if line.endswith(" ok"):
+ status[line.split()[0]] = "PASSED"
+ return status
+
+
+def matplotlib_log(log: str, config: dict) -> dict[str, str]:
+ return pytest_log(log.replace("MouseButton.LEFT", "1").replace("MouseButton.RIGHT", "3"), config)
+
+
+PARSERS = {
+ "astropy/astropy": pytest_v2_log,
+ "django/django": django_log,
+ "matplotlib/matplotlib": matplotlib_log,
+ "mwaskom/seaborn": seaborn_log,
+ "pallets/flask": pytest_log,
+ "psf/requests": pytest_options_log,
+ "pydata/xarray": pytest_log,
+ "pylint-dev/pylint": pytest_options_log,
+ "pytest-dev/pytest": pytest_log,
+ "scikit-learn/scikit-learn": pytest_v2_log,
+ "sphinx-doc/sphinx": pytest_v2_log,
+ "sympy/sympy": sympy_log,
+}
+
+
+def names(config: dict, field: str) -> list[str]:
+ value = config.get(field, [])
+ value = json.loads(value) if isinstance(value, str) else value
+ if not isinstance(value, list) or not all(isinstance(item, str) for item in value):
+ raise ValueError(f"invalid {field}")
+ return value
+
+
+def grade(config: dict, log: str) -> dict:
+ instance = config.get("instance_id")
+ repo = config.get("repo")
+ if not isinstance(instance, str) or repo not in PARSERS:
+ raise ValueError("unsupported SWE-bench task identity")
+ status = PARSERS[repo](log, config)
+ expected = {"FAIL_TO_PASS": names(config, "FAIL_TO_PASS"), "PASS_TO_PASS": names(config, "PASS_TO_PASS")}
+ expected_names = set(expected["FAIL_TO_PASS"] + expected["PASS_TO_PASS"])
+ missing = expected_names.difference(status)
+ if missing:
+ raise ValueError(f"test log is missing {len(missing)} expected SWE-bench statuses")
+ tests_status = {
+ "FAIL_TO_FAIL": {"success": [], "failure": []},
+ "PASS_TO_FAIL": {"success": [], "failure": []},
+ }
+ resolved = True
+ for group, tests in expected.items():
+ passed = [test for test in tests if status.get(test) in {"PASSED", "XFAIL"}]
+ failed = [test for test in tests if test not in passed]
+ tests_status[group] = {"success": passed, "failure": failed}
+ resolved = resolved and not failed
+ return {
+ instance: {
+ "patch_is_None": False,
+ "patch_exists": True,
+ "patch_successfully_applied": True,
+ "resolved": resolved,
+ "tests_status": tests_status,
+ }
+ }
diff --git a/scripts/evals/short_swe/oracle_harness.py b/scripts/evals/short_swe/oracle_harness.py
new file mode 100644
index 0000000000..12fef5a343
--- /dev/null
+++ b/scripts/evals/short_swe/oracle_harness.py
@@ -0,0 +1,74 @@
+"""Deterministic gold-patch harness for the SWE-bench scorer preflight.
+
+The embedded Astropy source excerpt is licensed under the BSD 3-Clause License:
+
+Copyright (c) 2011-2026, Astropy Developers. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+* Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+* Neither the name of the Astropy Team nor the names of its contributors may be
+ used to endorse or promote products derived from this software without
+ specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+"""
+
+from verifiers.v1.clients import ModelContext
+from verifiers.v1.configs.harness import HarnessConfig
+from verifiers.v1.harness import Harness
+from verifiers.v1.runtimes import ProgramResult, Runtime
+from verifiers.v1.task import TaskData
+from verifiers.v1.trace import Trace
+
+ORIGINAL = """ # Fail
+ raise AttributeError(
+ f"'{self.__class__.__name__}' object has no attribute '{attr}'"
+ )"""
+REPLACEMENT = """ # Call __getattribute__; this will give correct exception.
+ return self.__getattribute__(attr)"""
+SCRIPT = f"""from pathlib import Path
+path = Path("/testbed/astropy/coordinates/sky_coordinate.py")
+source = path.read_text()
+old = {ORIGINAL!r}
+new = {REPLACEMENT!r}
+assert source.count(old) == 1
+path.write_text(source.replace(old, new))
+"""
+
+
+class OracleHarnessConfig(HarnessConfig):
+ pass
+
+
+class OracleHarness(Harness[OracleHarnessConfig]):
+ async def launch(
+ self,
+ ctx: ModelContext,
+ trace: Trace,
+ runtime: Runtime,
+ endpoint: str,
+ secret: str,
+ mcp_urls: dict[str, str],
+ data: TaskData,
+ ) -> ProgramResult:
+ path = "/tmp/prime-agent-swebench-oracle.py"
+ await runtime.write(path, SCRIPT.encode())
+ return await runtime.run(["python", path], {})
+
+
+__all__ = ["OracleHarness"]
diff --git a/scripts/evals/short_swe/prepare.py b/scripts/evals/short_swe/prepare.py
new file mode 100644
index 0000000000..d9dcf1fbc7
--- /dev/null
+++ b/scripts/evals/short_swe/prepare.py
@@ -0,0 +1,297 @@
+#!/usr/bin/env python3
+"""Validate immutable inputs and generate exact Short SWE Verifiers configs."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import subprocess
+import tomllib
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent
+SHA_RE = re.compile(r"^[0-9a-f]{40}$")
+DIGEST_RE = re.compile(r"^[0-9a-f]{64}$")
+CANDIDATE_TARBALLS = {
+ f"{name}-0.0.0-benchmark.tgz"
+ for name in ("prime-agent", "prime-agent-ai", "prime-agent-core", "prime-agent-tui")
+}
+
+
+def revision(path: Path) -> str:
+ return subprocess.run(
+ ["git", "rev-parse", "HEAD"],
+ cwd=path,
+ check=True,
+ capture_output=True,
+ text=True,
+ ).stdout.strip()
+
+
+def replace_once(path: Path, old: str, new: str) -> None:
+ text = path.read_text()
+ if text.count(old) != 1:
+ raise ValueError(f"trusted compatibility patch did not match {path}")
+ path.write_text(text.replace(old, new))
+
+
+def strip_task_runtime_credentials(verifiers: Path) -> None:
+ """Patch the pinned Harbor task boundary before it populates a sandbox environment."""
+ harbor = verifiers / "verifiers/v1/tasksets/harbor/taskset.py"
+ old = " def runtime_env(self) -> dict[str, str]:\n return resolve_env(self.data.env)"
+ new = (
+ " def runtime_env(self) -> dict[str, str]:\n"
+ " env = resolve_env(self.data.env)\n"
+ ' for name in ("PRIME_API_KEY", "PRIME_SANDBOX_API_KEY", '
+ '"GITHUB_TOKEN", "GH_TOKEN", "HF_TOKEN"):\n'
+ " env.pop(name, None)\n"
+ " return env"
+ )
+ replace_once(harbor, old, new)
+
+
+def install_verified_verifier(manifest: dict, environments: Path) -> None:
+ # Tests run in a fresh, network-free verifier. Their bounded log is parsed here,
+ # outside candidate control, against the trusted task package's config.
+ by_id = {item["id"]: item for item in manifest["tasksets"]}
+ package = environments / by_id["swebench-verified"]["package"] / "swebench_verified"
+ taskset = package / "taskset.py"
+ replace_once(
+ taskset,
+ "`SWEBenchVerifiedTask` (a `HarborTask` whose `finalize` ensures uv before harbor's verifier runs).",
+ "`SWEBenchVerifiedTask` (a `HarborTask` graded in a fresh offline verifier sandbox).",
+ )
+ replace_once(taskset, "from verifiers.v1.runtimes import Runtime\n", "")
+ replace_once(taskset, "from verifiers.v1.runtimes.base import _ENSURE_UV\n", "")
+ replace_once(
+ taskset,
+ "from verifiers.v1.tasksets.harbor import HarborConfig, HarborTask, HarborTaskset\n",
+ "from verifiers.v1.tasksets.harbor import HarborConfig, HarborTask, HarborTaskset\n"
+ "from verifiers.v1.tasksets.harbor.taskset import CollectHook, VerifierConfig\n"
+ "from verifiers.v1.utils.artifacts import Artifact\n"
+ "from secure_harbor import SecureVerifiedMixin\n"
+ "from verified_verifier import patch_collect_command\n",
+ )
+ old = """class SWEBenchVerifiedTask(HarborTask):
+ async def finalize(self, trace: vf.Trace, runtime: Runtime) -> None:
+ # The SWE-bench verifier runs `uv run parser.py` but never installs uv, relying on the
+ # harness to leave one on PATH. rlm pins uv off PATH, so the grader hits `uv: command not
+ # found` and scores 0 even for correct fixes. Ensure uv before scoring, under any harness.
+ await runtime.run(["sh", "-c", _ENSURE_UV], {})
+ await super().finalize(trace, runtime)
+"""
+ new = """class SWEBenchVerifiedTask(SecureVerifiedMixin, HarborTask):
+ pass
+"""
+ replace_once(taskset, old, new)
+ old_data = 'data = task.data.model_copy(update={"image": image, "workdir": "/testbed"})'
+ new_data = """if task.data.artifacts or task.data.collect or task.data.verifier is not None:
+ raise ValueError(f"{task.data.name}: unexpected verifier transfer configuration")
+ data = task.data.model_copy(
+ update={
+ "image": image,
+ "workdir": "/testbed",
+ "artifacts": [Artifact(source="/tmp/prime-agent.patch")],
+ "collect": [
+ CollectHook(
+ command=patch_collect_command(Path(task.data.task_dir))
+ )
+ ],
+ "verifier": VerifierConfig(fresh_copy=True, network_allow=[]),
+ }
+ )"""
+ replace_once(taskset, old_data, new_data)
+
+
+def pin_taskset_sources(manifest: dict, verifiers: Path, environments: Path) -> None:
+ by_id = {item["id"]: item for item in manifest["tasksets"]}
+ for taskset_id, module in (
+ ("swebench-verified", "swebench_verified"),
+ ("swebench-pro", "swebench_pro"),
+ ):
+ path = environments / by_id[taskset_id]["package"] / module / "taskset.py"
+ unpinned = by_id[taskset_id]["dataset"].split("@", 1)[0]
+ pinned = by_id[taskset_id]["dataset"]
+ text = path.read_text()
+ count = text.count(f'"{unpinned}"')
+ if count != 2:
+ raise ValueError(f"dataset pin did not match {path}")
+ path.write_text(text.replace(f'"{unpinned}"', f'"{pinned}"'))
+
+ scale = environments / by_id["scaleswe"]["package"] / "scaleswe/taskset.py"
+ old = "dataset = load_dataset(self.config.dataset_name, split=self.config.split)"
+ pin = manifest["scaleswe_dataset_revision"]
+ new = (
+ "dataset = load_dataset(\n"
+ " self.config.dataset_name,\n"
+ " split=self.config.split,\n"
+ f' revision="{pin}",\n'
+ " )"
+ )
+ replace_once(scale, old, new)
+
+ install_verified_verifier(manifest, environments)
+ strip_task_runtime_credentials(verifiers)
+
+ # This Verifiers revision eagerly imports the optional NeMo Gym plugin from
+ # tasksets.__init__. Harbor does not need it, and MCP 2.0 no longer exposes
+ # that plugin's legacy import path. Keep this compatibility patch narrow.
+ init = verifiers / "verifiers/v1/tasksets/__init__.py"
+ text = init.read_text()
+ block = "from verifiers.v1.tasksets.nemo_gym import NeMoGymConfig, NeMoGymTaskset\n"
+ if block in text:
+ text = text.replace(block, "")
+ text = text.replace(' "NeMoGymConfig",\n "NeMoGymTaskset",\n', "")
+ init.write_text(text)
+
+
+def validate_verifiers_lock(manifest: dict, verifiers: Path) -> None:
+ lock = tomllib.loads((verifiers / "uv.lock").read_text())
+ versions = {
+ package.get("version") for package in lock.get("package", []) if package.get("name") == "harbor"
+ }
+ if versions != {manifest.get("harbor_version")}:
+ raise ValueError("Verifiers Harbor version does not match the manifest")
+
+
+def validate_manifest(manifest: dict) -> None:
+ expected_limits = {
+ "max_turns": 128,
+ "max_output_tokens": 100_000,
+ "max_total_tokens": 5_000_000,
+ "rollout_timeout_seconds": 3_600,
+ "max_model_request_bytes": 16_000_000,
+ }
+ if (
+ manifest.get("schema_version") != 1
+ or manifest.get("autonomous") is not False
+ or manifest.get("network_policy") != "framework-only"
+ or manifest.get("limits") != expected_limits
+ ):
+ raise ValueError("unsupported Short SWE manifest")
+ if not all(isinstance(manifest.get(key), str) and manifest[key] for key in ("model", "backup_model")):
+ raise ValueError("Short SWE must pin the model and its backup")
+ if not SHA_RE.fullmatch(manifest.get("verifiers_commit", "")):
+ raise ValueError("invalid Verifiers revision")
+ if not SHA_RE.fullmatch(manifest.get("environments_commit", "")):
+ raise ValueError("invalid environments revision")
+ expected = {"swebench-verified": 15, "swebench-pro": 8, "scaleswe": 5}
+ tasksets = manifest.get("tasksets", [])
+ actual = {item.get("id"): len(item.get("tasks", [])) for item in tasksets}
+ if actual != expected:
+ raise ValueError("Short SWE must contain the fixed 15/8/5 task slices")
+ scaleswe = next(item for item in tasksets if item["id"] == "scaleswe")
+ if scaleswe.get("filter_unavailable_images") is not False:
+ raise ValueError("Scale-SWE image filtering must remain disabled")
+ tasks = [task for item in tasksets for task in item["tasks"]]
+ if len(tasks) != 28 or len(tasks) != len(set(tasks)):
+ raise ValueError("Short SWE task keys must be 28 unique names")
+
+
+def config_text(item: dict) -> str:
+ """Minimal TOML: the hosted flow globs these file stems, never their content."""
+ return f"id = {json.dumps(item['id'])}\n"
+
+
+def toml_array(values: list[str]) -> str:
+ return "[" + ", ".join(json.dumps(value) for value in values) + "]"
+
+
+def oracle_config_text(manifest: dict) -> str:
+ return "\n".join(
+ [
+ f"model = {json.dumps(manifest['model'])}",
+ "num_rollouts = 1",
+ "max_concurrent = 1",
+ "",
+ "[env]",
+ 'id = "secure_harbor"',
+ "",
+ "[env.timeout]",
+ "finalize = 600",
+ "",
+ "[env.verifier]",
+ "retries = 0",
+ "",
+ "[env.verifier.runtime]",
+ 'type = "prime"',
+ "allow = []",
+ "vm = true",
+ "labels = "
+ + toml_array(
+ [
+ "prime-agent-behavioral-v1",
+ f"repository:{os.environ.get('GITHUB_REPOSITORY', 'local/local')}",
+ f"run:{os.environ.get('GITHUB_RUN_ID', 'local')}",
+ f"attempt:{os.environ.get('GITHUB_RUN_ATTEMPT', 'local')}",
+ "role:oracle-verifier",
+ ]
+ ),
+ "",
+ "[env.agent]",
+ "max_turns = 4",
+ "max_output_tokens = 20000",
+ "max_total_tokens = 100000",
+ "",
+ "[env.agent.timeout]",
+ "rollout = 600",
+ "scoring = 600",
+ "",
+ "[env.taskset]",
+ 'id = "swebench-verified"',
+ 'tasks = ["astropy__astropy-14096"]',
+ "",
+ "[env.agent.harness]",
+ 'id = "oracle_harness"',
+ "",
+ "[env.agent.runtime]",
+ 'type = "prime"',
+ "allow = []",
+ "labels = "
+ + toml_array(
+ [
+ "prime-agent-behavioral-v1",
+ f"repository:{os.environ.get('GITHUB_REPOSITORY', 'local/local')}",
+ f"run:{os.environ.get('GITHUB_RUN_ID', 'local')}",
+ f"attempt:{os.environ.get('GITHUB_RUN_ATTEMPT', 'local')}",
+ "role:oracle",
+ ]
+ ),
+ "",
+ ]
+ )
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--verifiers", required=True, type=Path)
+ parser.add_argument("--environments", required=True, type=Path)
+ parser.add_argument("--base-artifacts", required=True, type=Path)
+ parser.add_argument("--base-sha", required=True)
+ parser.add_argument("--head-artifacts", required=True, type=Path)
+ parser.add_argument("--head-sha", required=True)
+ parser.add_argument("--output", required=True, type=Path)
+ args = parser.parse_args()
+ manifest = json.loads((ROOT / "short-swe.json").read_text())
+ validate_manifest(manifest)
+ if revision(args.verifiers) != manifest["verifiers_commit"]:
+ raise ValueError("Verifiers checkout does not match the manifest")
+ if revision(args.environments) != manifest["environments_commit"]:
+ raise ValueError("environments checkout does not match the manifest")
+ validate_verifiers_lock(manifest, args.verifiers)
+ if not SHA_RE.fullmatch(args.base_sha) or not SHA_RE.fullmatch(args.head_sha):
+ raise ValueError("invalid evaluated revision")
+ pin_taskset_sources(manifest, args.verifiers, args.environments)
+ args.output.mkdir(parents=True, exist_ok=True)
+ (args.output / "oracle.toml").write_text(oracle_config_text(manifest))
+ for side in ("base", "head"):
+ target = args.output / side
+ target.mkdir(parents=True, exist_ok=True)
+ for item in manifest["tasksets"]:
+ (target / f"{item['id']}.toml").write_text(config_text(item))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/evals/short_swe/report.py b/scripts/evals/short_swe/report.py
new file mode 100644
index 0000000000..ce1b16a228
--- /dev/null
+++ b/scripts/evals/short_swe/report.py
@@ -0,0 +1,241 @@
+#!/usr/bin/env python3
+"""Validate and render the paired Short SWE release check."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+from collections import defaultdict
+from pathlib import Path
+
+MARKER = "<!-- prime-agent-behavioral-eval:v1 -->"
+TASK_COUNT = 28
+RESOLVED_LOSS_LIMIT = 5
+MODEL_FAILURE_LIMIT = 3
+RATIO_LIMIT = 2.0
+COLOR_THRESHOLD = 0.2
+TASKSET_LABELS = {
+ "swebench-verified": "SWE-bench Verified",
+ "swebench-pro": "SWE-bench Pro",
+ "scaleswe": "ScaleSWE",
+}
+
+
+def aggregate(tasks: list[dict]) -> dict:
+ fields = ("uncached_input_tokens", "cached_input_tokens", "output_tokens", "model_calls")
+ if len(tasks) != TASK_COUNT:
+ raise ValueError("a result side must contain 28 tasks")
+ for task in tasks:
+ if (
+ not isinstance(task.get("resolved"), bool)
+ or not isinstance(task.get("model_failure"), bool)
+ or task["resolved"]
+ and task["model_failure"]
+ ):
+ raise ValueError("invalid outcome value")
+ for field in fields:
+ value = task.get(field)
+ if not isinstance(value, int) or isinstance(value, bool) or value < 0:
+ raise ValueError(f"invalid {field}")
+ elapsed = task.get("e2e_seconds")
+ if (
+ not isinstance(elapsed, (int, float))
+ or isinstance(elapsed, bool)
+ or not math.isfinite(elapsed)
+ or elapsed < 0
+ ):
+ raise ValueError("invalid e2e_seconds")
+ return {
+ "tasks": len(tasks),
+ "resolved": sum(task["resolved"] for task in tasks),
+ "model_failures": sum(task["model_failure"] for task in tasks),
+ **{field: sum(task[field] for task in tasks) for field in fields},
+ "e2e_seconds": sum(task["e2e_seconds"] for task in tasks),
+ }
+
+
+def color(relative: float, regressed: bool) -> str:
+ muted, vivid = ((170, 106, 101), (229, 72, 77)) if regressed else ((102, 129, 109), (31, 146, 78))
+ intensity = min(abs(relative), 1.0)
+ channels = (round(start + (end - start) * intensity) for start, end in zip(muted, vivid, strict=True))
+ return "#" + "".join(f"{channel:02x}" for channel in channels)
+
+
+def change(
+ head: float, base: float, *, lower_is_better: bool = True, threshold: float = COLOR_THRESHOLD
+) -> str:
+ delta = head - base
+ if delta == 0:
+ return "≈ 0 (0.0%)"
+ relative = delta / base if base else None
+ style_change = relative if relative is not None else 1.0
+ percentage = "n/a" if relative is None else f"{relative * 100:+.1f}%"
+ text = f"{delta:+,.1f} ({percentage})" if isinstance(delta, float) else f"{delta:+,} ({percentage})"
+ if abs(style_change) < threshold:
+ return f"≈ {text}"
+ regressed = delta > 0 if lower_is_better else delta < 0
+ arrow = "↑" if delta > 0 else "↓"
+ safe = f"{arrow} {text}".replace("%", r"\%")
+ return rf"$`\textcolor{{{color(style_change, regressed)}}}{{\textsf{{{safe}}}}}`$"
+
+
+def compare(base: dict, head: dict) -> list[str]:
+ findings = []
+ resolved_delta = head["resolved"] - base["resolved"]
+ if resolved_delta <= -RESOLVED_LOSS_LIMIT:
+ findings.append(f"Resolved tasks decreased by {-resolved_delta}.")
+ additional_failures = head["model_failures"] - base["model_failures"]
+ if additional_failures >= MODEL_FAILURE_LIMIT:
+ findings.append(f"Model failures increased by {additional_failures}.")
+ for field, label in (("output_tokens", "Output tokens"), ("e2e_seconds", "Cumulative task time")):
+ ratio = head[field] / base[field] if base[field] > 0 else math.inf
+ if resolved_delta <= 0 and (
+ (base[field] == 0 and head[field] > 0) or (base[field] > 0 and ratio >= RATIO_LIMIT)
+ ):
+ findings.append(f"{label} reached {ratio:.2f}x base without more resolutions.")
+ return findings
+
+
+def grouped(tasks: list[dict]) -> dict[str, dict]:
+ values = defaultdict(list)
+ for task in tasks:
+ values[task["taskset"]].append(task)
+ return {name: aggregate_side(rows) for name, rows in values.items()}
+
+
+def aggregate_side(tasks: list[dict]) -> dict:
+ return {
+ "tasks": len(tasks),
+ "resolved": sum(task["resolved"] for task in tasks),
+ "uncached_input_tokens": sum(task["uncached_input_tokens"] for task in tasks),
+ "cached_input_tokens": sum(task["cached_input_tokens"] for task in tasks),
+ "output_tokens": sum(task["output_tokens"] for task in tasks),
+ }
+
+
+def validate_result(result: dict, request: dict) -> tuple[dict, dict]:
+ pinned = {
+ "internal/glm-5.3-fast",
+ "internal/deepseek-v4.1-flash",
+ }
+ if (
+ result.get("schema_version") != 1
+ or result.get("request") != request
+ or result.get("model") not in pinned
+ ):
+ raise ValueError("result identity does not match the trusted request")
+ sides = result.get("sides")
+ if not isinstance(sides, dict) or set(sides) != {"base", "head"}:
+ raise ValueError("paired result is incomplete")
+ base, head = aggregate(sides["base"]), aggregate(sides["head"])
+ base_ids = {(task["taskset"], task["task"]) for task in sides["base"]}
+ head_ids = {(task["taskset"], task["task"]) for task in sides["head"]}
+ if len(base_ids) != TASK_COUNT or base_ids != head_ids:
+ raise ValueError("base and head task identities do not match")
+ return base, head
+
+
+def render(result: dict, request: dict, evaluations: dict | None = None) -> tuple[str, str]:
+ base, head = validate_result(result, request)
+ findings = compare(base, head)
+ verdict = "fail" if findings else "pass"
+ run_url = f"https://github.com/{request['repository']}/actions/runs/{request['run_id']}"
+ resolution_change = change(head["resolved"], base["resolved"], lower_is_better=False, threshold=0)
+ failure_change = change(head["model_failures"], base["model_failures"], threshold=0)
+ uncached_change = change(head["uncached_input_tokens"], base["uncached_input_tokens"])
+ cached_change = change(head["cached_input_tokens"], base["cached_input_tokens"])
+ output_change = change(head["output_tokens"], base["output_tokens"])
+ elapsed_change = change(head["e2e_seconds"], base["e2e_seconds"])
+ lines = [
+ MARKER,
+ f"<!-- head:{request['head_sha']} -->",
+ f"### Behavioral evaluation — {verdict}",
+ "",
+ f"PR head `{request['head_sha'][:8]}` compared with exact base `{request['base_sha'][:8]}`.",
+ f"Model `{result['model']}`. Inference cost: **$0**.",
+ "Time is summed across task traces; concurrent tasks overlap in wall-clock time.",
+ "",
+ "| Metric | Exact base | PR head | Change |",
+ "| --- | ---: | ---: | ---: |",
+ f"| Resolution | {base['resolved']}/{TASK_COUNT} | {head['resolved']}/{TASK_COUNT} | "
+ f"{resolution_change} |",
+ f"| Model failures | {base['model_failures']} | {head['model_failures']} | {failure_change} |",
+ f"| Uncached input tokens | {base['uncached_input_tokens']:,} | "
+ f"{head['uncached_input_tokens']:,} | {uncached_change} |",
+ f"| Cached input tokens | {base['cached_input_tokens']:,} | "
+ f"{head['cached_input_tokens']:,} | {cached_change} |",
+ f"| Output tokens | {base['output_tokens']:,} | {head['output_tokens']:,} | {output_change} |",
+ f"| Cumulative task time | {base['e2e_seconds']:,.1f} s | {head['e2e_seconds']:,.1f} s | "
+ f"{elapsed_change} |",
+ "",
+ "#### Results by taskset",
+ "",
+ "| Taskset | Side | Resolution | Uncached input | Cached input | Output |",
+ "| --- | --- | ---: | ---: | ---: | ---: |",
+ ]
+ tasksets = {side: grouped(result["sides"][side]) for side in ("base", "head")}
+ if tasksets["base"].keys() != tasksets["head"].keys():
+ raise ValueError("base and head tasksets do not match")
+ for name in TASKSET_LABELS:
+ for side, label in (("base", "Exact base"), ("head", "PR head")):
+ value = tasksets[side][name]
+ lines.append(
+ f"| {TASKSET_LABELS[name]} | {label} | {value['resolved']}/{value['tasks']} | "
+ f"{value['uncached_input_tokens']:,} | {value['cached_input_tokens']:,} | "
+ f"{value['output_tokens']:,} |"
+ )
+ lines.extend(["", "**Threshold findings:**" if findings else "No drastic threshold crossed.", ""])
+ lines.extend(f"- {finding}" for finding in findings)
+ lines.append(f"[Workflow run]({run_url})")
+ if evaluations:
+ lines.extend(["", "**Hosted evaluations (Prime Evals):**", ""])
+ for key in sorted(evaluations):
+ record = evaluations[key]
+ evaluation_id = record.get("evaluation_id", "")
+ viewer = f"https://app.primeintellect.ai/dashboard/evaluations/{evaluation_id}"
+ lines.append(f"- {key}: [{record.get('name', evaluation_id)}]({viewer})")
+ lines.append("")
+ return "\n".join(lines), verdict
+
+
+def render_failure(request: dict) -> str:
+ run_url = f"https://github.com/{request['repository']}/actions/runs/{request['run_id']}"
+ return "\n".join(
+ [
+ MARKER,
+ f"<!-- head:{request['head_sha']} -->",
+ "### Behavioral evaluation — failed",
+ "",
+ "The exact base/head comparison did not produce a complete validated result.",
+ "Missing or malformed measurements are never treated as improvements.",
+ "",
+ f"[Run, logs, and artifacts]({run_url})",
+ "",
+ ]
+ )
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--request", required=True, type=Path)
+ parser.add_argument("--result", required=True, type=Path)
+ parser.add_argument("--markdown", required=True, type=Path)
+ parser.add_argument("--verdict", required=True, type=Path)
+ parser.add_argument("--evaluations", type=Path, default=None)
+ args = parser.parse_args()
+ request = json.loads(args.request.read_text())
+ evaluations = None
+ if args.evaluations and args.evaluations.exists():
+ evaluations = json.loads(args.evaluations.read_text())
+ try:
+ markdown, verdict = render(json.loads(args.result.read_text()), request, evaluations)
+ except (OSError, ValueError, TypeError, KeyError, json.JSONDecodeError):
+ markdown, verdict = render_failure(request), "fail"
+ args.markdown.parent.mkdir(parents=True, exist_ok=True)
+ args.markdown.write_text(markdown)
+ args.verdict.write_text(verdict + "\n")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/evals/short_swe/secure_harbor.py b/scripts/evals/short_swe/secure_harbor.py
new file mode 100644
index 0000000000..6ed2c03c91
--- /dev/null
+++ b/scripts/evals/short_swe/secure_harbor.py
@@ -0,0 +1,104 @@
+"""Harbor environment that preserves specialized tasks in a separate verifier."""
+
+import json
+import time
+from pathlib import Path
+
+import verifiers.v1 as vf
+from offline_swebench_grader import grade
+from verified_verifier import rewrite_test_script, trusted_base_commit
+from verifiers.v1.runtimes import Runtime
+from verifiers.v1.tasksets.harbor import HarborEnv, HarborEnvConfig, HarborTask
+from verifiers.v1.tasksets.harbor.taskset import verifier_box_data
+
+
+class SecureVerifiedMixin:
+ async def setup(self, runtime: Runtime) -> None:
+ await super().setup(runtime)
+ if not self.data.name.endswith(" (verifier)"):
+ return
+ base = trusted_base_commit(Path(self.data.task_dir))
+ result = await runtime.run(
+ ["sh", "-c", f"git reset --hard {base} && git clean -fd"],
+ {},
+ )
+ if result.exit_code:
+ detail = (result.stderr or result.stdout).strip()[-2_000:]
+ raise RuntimeError(f"could not reset SWE-bench verifier to its trusted base: {detail}")
+
+ async def stage_verifier(self, runtime: Runtime) -> None:
+ await super().stage_verifier(runtime)
+ result = await runtime.run(
+ ["sh", "-c", "test ! -s /tmp/prime-agent.patch || git apply --binary /tmp/prime-agent.patch"],
+ {},
+ )
+ if result.exit_code:
+ detail = (result.stderr or result.stdout).strip()[-2_000:]
+ raise RuntimeError(f"candidate patch did not apply in verifier: {detail}")
+
+ async def run_verifier(self, runtime: Runtime, trace: vf.Trace) -> float:
+ script = rewrite_test_script((await runtime.read("/tests/test.sh", max_bytes=2_000_000)).decode())
+ await runtime.write("/tests/test.sh", script.encode())
+ removed = await runtime.run(
+ ["rm", "-f", "/tests/config.json", "/tmp/tests.tgz"],
+ {},
+ )
+ if removed.exit_code:
+ raise RuntimeError("could not remove verifier-only metadata before tests")
+ result = await runtime.run(
+ [
+ "bash",
+ "-c",
+ "set -o pipefail; bash /tests/test.sh 2>&1 | head -c 16000001",
+ ],
+ {"PIP_DISABLE_PIP_VERSION_CHECK": "1", "PIP_NO_INDEX": "1"},
+ )
+ log = result.stdout
+ try:
+ if len(log.encode()) > 16_000_000:
+ raise ValueError("verifier output exceeds its size limit")
+ config = json.loads((Path(self.data.task_dir) / "tests" / "config.json").read_text())
+ instance = config["instance_id"]
+ record = grade(config, log)[instance]
+ resolved = record["resolved"]
+ if not isinstance(resolved, bool) or result.exit_code not in (0, 1):
+ raise ValueError("inconsistent verifier result")
+ except (KeyError, OSError, UnicodeDecodeError, ValueError, json.JSONDecodeError) as exc:
+ detail = (log or result.stderr or result.stdout).strip()[-2_000:]
+ raise RuntimeError(f"SWE-bench verifier failed closed: {detail}") from exc
+ trace.info["swebench_verifier"] = record
+ trace.info["swebench_verifier_log_tail"] = log[-100_000:]
+ return float(resolved)
+
+
+class SecureHarborConfig(HarborEnvConfig):
+ pass
+
+
+class SecureHarbor(HarborEnv):
+ async def finalize(self, task: vf.Task, episode: vf.Episode) -> None:
+ if not isinstance(task, HarborTask) or task.data.verifier is None:
+ return
+ solution = episode.traces[0]
+ if not solution.ok:
+ return
+ artifacts = solution.state.artifacts
+ expected = {"/logs/artifacts", "/tmp/prime-agent.patch"}
+ if set(artifacts) != expected or artifacts["/logs/artifacts"] is not None:
+ raise RuntimeError("solver produced undeclared verifier artifacts")
+ grader = type(task)(verifier_box_data(task.data), task.config)
+ started = time.monotonic()
+ scores, solution = await self.grade(
+ self.verifier_config(task),
+ grader,
+ solution,
+ scoring_timeout_covers_attempt=True,
+ )
+ solution.info["isolated_verifier_seconds"] = time.monotonic() - started
+ items = scores.items() if isinstance(scores, dict) else [("solved", scores)]
+ for name, value in items:
+ solution.record_reward(name, value)
+ episode.traces[0] = solution
+
+
+__all__ = ["SecureHarbor"]
diff --git a/scripts/evals/short_swe/short-swe.json b/scripts/evals/short_swe/short-swe.json
new file mode 100644
index 0000000000..e928976015
--- /dev/null
+++ b/scripts/evals/short_swe/short-swe.json
@@ -0,0 +1,72 @@
+{
+ "schema_version": 1,
+ "model": "internal/glm-5.3-fast",
+ "autonomous": false,
+ "max_concurrent": 4,
+ "num_rollouts": 1,
+ "verifiers_commit": "9df6a3c01bb640c34380f4f6dff63a1b168dee22",
+ "environments_commit": "a41660014674ba43d56ed054199dde25ddf69946",
+ "harbor_version": "0.21.0",
+ "scaleswe_dataset_revision": "8935f8e55244fd56080cdb8dcd0819a57e8a003c",
+ "tasksets": [
+ {
+ "id": "swebench-verified",
+ "dataset": "swe-bench/swe-bench-verified@sha256:b934b0cc3dc800fe945eaf9f1623329db97ee3133c706d20644524c7759fb341",
+ "package": "environments/swe/swebench_verified",
+ "tasks": [
+ "astropy__astropy-12907",
+ "django__django-16642",
+ "django__django-14034",
+ "matplotlib__matplotlib-26342",
+ "mwaskom__seaborn-3187",
+ "pallets__flask-5014",
+ "psf__requests-5414",
+ "pydata__xarray-6992",
+ "pylint-dev__pylint-8898",
+ "pytest-dev__pytest-7205",
+ "scikit-learn__scikit-learn-14629",
+ "sphinx-doc__sphinx-7462",
+ "sphinx-doc__sphinx-11445",
+ "sympy__sympy-16450",
+ "sympy__sympy-22914"
+ ]
+ },
+ {
+ "id": "swebench-pro",
+ "dataset": "scale-ai/swe-bench-pro@sha256:88411d32ff27e53a4c1a7e29f0c2aeba180c8e5d60f221cab5ed56325f33549d",
+ "package": "environments/swe/swebench_pro",
+ "tasks": [
+ "instance_element-hq__element-web-33e8edb3d508d6eefb354819ca693b7accc695e7",
+ "instance_internetarchive__openlibrary-111347e9583372e8ef91c82e0612ea437ae3a9c9-v2d9a6c849c60ed19fd0858ce9e40b7cc8e097e59",
+ "instance_qutebrowser__qutebrowser-44e64199ed38003253f0296badd4a447645067b6-v2ef375ac784985212b1805e1d0431dc8f1b3c171",
+ "instance_nodebb__nodebb-0f788b8eaa4bba3c142d171fd941d015c53b65fc-v0ec6d6c2baf3cb4797482ce4829bc25cd5716649",
+ "instance_ansible__ansible-a6e671db25381ed111bbad0ab3e7d97366395d05-v0f01c69f1e2528b935359cfe578530722bca2c59",
+ "instance_gravitational__teleport-0ecf31de0e98b272a6a2610abe1bbedd379a38a3-vce94f93ad1030e3136852817f2423c1b3ac37bc4",
+ "instance_navidrome__navidrome-8383527aaba1ae8fa9765e995a71a86c129ef626",
+ "instance_tutao__tutanota-09c2776c0fce3db5c6e18da92b5a45dce9f013aa-vbc0d9ba8f0071fbe982809910959a6ff8884dbbf"
+ ]
+ },
+ {
+ "id": "scaleswe",
+ "dataset": "PrimeIntellect/Scale-SWE-Verified",
+ "package": "environments/swe/scaleswe",
+ "tasks": [
+ "auth0_auth0-python_pr671",
+ "beetbox_beets_pr3661",
+ "adamtheturtle_doccmd_pr42",
+ "arviz-devs_preliz_pr249",
+ "beetbox_beets_pr4386"
+ ],
+ "filter_unavailable_images": false
+ }
+ ],
+ "network_policy": "framework-only",
+ "limits": {
+ "max_turns": 128,
+ "max_output_tokens": 100000,
+ "max_total_tokens": 5000000,
+ "rollout_timeout_seconds": 3600,
+ "max_model_request_bytes": 16000000
+ },
+ "backup_model": "internal/deepseek-v4.1-flash"
+}
diff --git a/scripts/evals/short_swe/tests/test_hosted_eval.py b/scripts/evals/short_swe/tests/test_hosted_eval.py
new file mode 100644
index 0000000000..44c374c0a2
--- /dev/null
+++ b/scripts/evals/short_swe/tests/test_hosted_eval.py
@@ -0,0 +1,324 @@
+"""Focused tests for the hosted launch/wait/collect orchestration."""
+
+import json
+import re
+import sys
+import time
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[4]
+sys.path.insert(0, str(ROOT))
+
+from scripts.evals.short_swe import hosted_eval # noqa: E402
+
+
+class FakePopen:
+ def __init__(self, command, **kwargs):
+ self.command = command
+
+
+def _write_inputs(tmp_path: Path) -> None:
+ request = {
+ "pr": 2306,
+ "head_sha": "c" * 40,
+ "base_sha": "b" * 40,
+ "repository": "PrimeIntellect-ai/prime-agent",
+ "run_id": 7,
+ }
+ sources = {
+ "base": {
+ "tarballs_url": "https://example.invalid/base.zip",
+ "commit": "b" * 40,
+ "checksums": {"prime-agent-0.0.0-benchmark.tgz": "0" * 64},
+ "token": "tok",
+ },
+ "head": {
+ "tarballs_url": "https://example.invalid/head.zip",
+ "commit": "c" * 40,
+ "checksums": {"prime-agent-0.0.0-benchmark.tgz": "1" * 64},
+ "token": "tok",
+ },
+ }
+ (tmp_path / "request.json").write_text(json.dumps(request))
+ (tmp_path / "sources.json").write_text(json.dumps(sources))
+
+
+def test_launch_starts_six_evaluations_with_the_manifest_identity(tmp_path: Path, monkeypatch) -> None:
+ _write_inputs(tmp_path)
+ commands = []
+ logs = {}
+
+ def fake_popen(command, **kwargs):
+ commands.append(command)
+ log = kwargs["stdout"]
+ name = command[command.index("--eval-name") + 1]
+ logs[name] = log
+ log.write(f"Evaluation ID: id-{name}\n")
+ log.flush()
+ return FakePopen(command)
+
+ monkeypatch.setattr(hosted_eval.subprocess, "Popen", fake_popen)
+ monkeypatch.setenv(hosted_eval.JOB_DEADLINE_ENV, str(time.time() + 300 * 60))
+ args = type(
+ "Args",
+ (),
+ {
+ "request": str(tmp_path / "request.json"),
+ "sources": str(tmp_path / "sources.json"),
+ "output": str(tmp_path / "hosted"),
+ "model": None,
+ },
+ )()
+ hosted_eval.launch(args)
+
+ assert len(commands) == 6
+ runs = json.loads((tmp_path / "hosted" / "hosted-runs.json").read_text())
+ assert set(runs) == {
+ "base/swebench-verified",
+ "base/swebench-pro",
+ "base/scaleswe",
+ "head/swebench-verified",
+ "head/swebench-pro",
+ "head/scaleswe",
+ }
+ for command in commands:
+ assert command[command.index("-m") + 1] == "internal/glm-5.3-fast"
+ assert command[command.index("--max-concurrent") + 1] == "4"
+ assert command[command.index("--timeout-minutes") + 1] == str(
+ hosted_eval.HOSTED_EVALUATION_TIMEOUT_MINUTES
+ )
+ assert command[command.index("-r") + 1] == "1"
+ secrets = json.loads(command[command.index("--custom-secrets") + 1])
+ assert set(secrets) == {
+ "CANDIDATE_TARBALLS_URL",
+ "CANDIDATE_COMMIT",
+ "CANDIDATE_CHECKSUMS",
+ "CANDIDATE_TOKEN",
+ }
+ assert secrets["CANDIDATE_COMMIT"] in ("b" * 40, "c" * 40)
+ for record in runs.values():
+ assert record["evaluation_id"].startswith("id-")
+ for log in logs.values():
+ log.close()
+
+
+def test_launch_rejects_an_unpinned_model(tmp_path: Path, monkeypatch) -> None:
+ _write_inputs(tmp_path)
+ monkeypatch.setattr(hosted_eval.subprocess, "Popen", lambda *a, **k: None)
+ args = type(
+ "Args",
+ (),
+ {
+ "request": str(tmp_path / "request.json"),
+ "sources": str(tmp_path / "sources.json"),
+ "output": str(tmp_path / "hosted"),
+ "model": "unrelated/model",
+ },
+ )()
+ try:
+ hosted_eval.launch(args)
+ raise AssertionError("unpinned model accepted")
+ except ValueError as error:
+ assert "not pinned" in str(error)
+
+
+def test_collect_writes_native_episodes(tmp_path: Path, monkeypatch) -> None:
+ runs = {
+ "base/swebench-verified": {
+ "evaluation_id": "id-1",
+ "num_examples": 1,
+ }
+ }
+ output = tmp_path / "hosted"
+ output.mkdir()
+ (output / "hosted-runs.json").write_text(json.dumps(runs))
+ sample = {"info": {"native_wrapper": {"id": "episode-1", "traces": []}}}
+
+ def fake_run(command, **kwargs):
+ return json.dumps({"samples": [sample]})
+
+ monkeypatch.setattr(hosted_eval, "run", fake_run)
+ args = type("Args", (), {"output": str(output)})()
+ hosted_eval.collect(args)
+
+ written = (output / "raw-eval" / "base" / "swebench-verified" / "traces.jsonl").read_text()
+ assert json.loads(written) == {"id": "episode-1", "traces": []}
+
+
+def test_collect_bounds_every_call_by_the_collection_reserve(tmp_path: Path, monkeypatch) -> None:
+ output = tmp_path / "hosted"
+ output.mkdir()
+ runs = {
+ "base/swebench-verified": {"evaluation_id": "id-1", "num_examples": 1},
+ "head/scaleswe": {"evaluation_id": "id-2", "num_examples": 1},
+ }
+ (output / "hosted-runs.json").write_text(json.dumps(runs))
+ sample = {"info": {"native_wrapper": {"id": "episode-1", "traces": []}}}
+ timeouts = []
+
+ def fake_run(command, **kwargs):
+ timeouts.append(kwargs.get("timeout"))
+ return json.dumps({"samples": [sample]})
+
+ monkeypatch.setattr(hosted_eval, "run", fake_run)
+ args = type("Args", (), {"output": str(output)})()
+ hosted_eval.collect(args)
+ assert len(timeouts) == 2
+ assert all(value and value <= hosted_eval.COLLECT_RESERVE_MINUTES * 60 for value in timeouts)
+
+ monkeypatch.setattr(hosted_eval, "COLLECT_RESERVE_MINUTES", 0)
+ with pytest.raises(RuntimeError, match="exceeded its reserve"):
+ hosted_eval.collect(args)
+
+
+def test_collect_fails_when_a_sample_lacks_the_native_episode(tmp_path: Path, monkeypatch) -> None:
+ runs = {"base/swebench-verified": {"evaluation_id": "id-1", "num_examples": 1}}
+ output = tmp_path / "hosted"
+ output.mkdir()
+ (output / "hosted-runs.json").write_text(json.dumps(runs))
+
+ def fake_run(command, **kwargs):
+ return json.dumps({"samples": [{"info": {}}]})
+
+ monkeypatch.setattr(hosted_eval, "run", fake_run)
+ args = type("Args", (), {"output": str(output)})()
+ try:
+ hosted_eval.collect(args)
+ raise AssertionError("missing native episode accepted")
+ except RuntimeError as error:
+ assert "native episode record" in str(error)
+
+
+def test_launch_uses_the_request_pr_key(tmp_path: Path) -> None:
+ # ci.py writes the request with a "pr" key; the launch name must use it.
+ source = hosted_eval.__file__
+ ci = (Path(source).parent / "ci.py").read_text()
+ assert '"pr"' in ci
+ hosted = Path(source).read_text()
+ assert "request['pr']" in hosted
+ assert "pull_request'" not in hosted
+
+
+def test_launch_stops_created_evaluations_when_a_spawn_fails(tmp_path: Path, monkeypatch) -> None:
+ _write_inputs(tmp_path)
+ stopped = []
+
+ def fake_popen(command, **kwargs):
+ if len(stopped) == 2: # third launch fails after two started
+ raise OSError("spawn failed")
+ stopped.append(None)
+ log = kwargs["stdout"]
+ name = command[command.index("--eval-name") + 1]
+ log.write(f"Evaluation ID: id-{name}\n")
+ log.flush()
+ return FakePopen(command)
+
+ def fake_run(command, **kwargs):
+ if command[:3] == ["prime", "eval", "stop"]:
+ stopped.append(command[3])
+ return "{}"
+
+ monkeypatch.setattr(hosted_eval.subprocess, "Popen", fake_popen)
+ monkeypatch.setattr(hosted_eval, "run", lambda *a, **k: "{}")
+ monkeypatch.setenv(hosted_eval.JOB_DEADLINE_ENV, str(time.time() + 300 * 60))
+ monkeypatch.setattr(hosted_eval.subprocess, "run", lambda command, **kwargs: fake_run(command))
+ args = type(
+ "Args",
+ (),
+ {
+ "request": str(tmp_path / "request.json"),
+ "sources": str(tmp_path / "sources.json"),
+ "output": str(tmp_path / "hosted"),
+ "model": None,
+ },
+ )()
+ try:
+ hosted_eval.launch(args)
+ raise AssertionError("partial launch did not fail")
+ except OSError:
+ pass
+ # Both already-created evaluations are stopped by ids parsed from their launcher logs.
+ assert stopped[:2] == [None, None]
+ assert len(stopped) == 4 and all(isinstance(value, str) for value in stopped[2:])
+
+
+def test_hosted_budgets_fit_the_github_job_limit(monkeypatch) -> None:
+ workflow = (ROOT / ".github" / "workflows" / "behavioral-evals.yml").read_text()
+ job_limit = int(re.search(r"\n timeout-minutes: (\d+)", workflow).group(1))
+ assert hosted_eval.GITHUB_JOB_LIMIT_MINUTES == job_limit
+ assert f"BEHAVIORAL_JOB_DEADLINE_EPOCH=$(( $(date +%s) + {job_limit} * 60 ))" in workflow
+ # The per-evaluation cap plus the collection reserve have to fit inside the job limit.
+ assert hosted_eval.HOSTED_EVALUATION_TIMEOUT_MINUTES + hosted_eval.COLLECT_RESERVE_MINUTES <= job_limit
+ budget_minutes = 100
+ monkeypatch.setenv(hosted_eval.JOB_DEADLINE_ENV, str(time.time() + budget_minutes * 60 + 1))
+ assert hosted_eval.evaluation_timeout_minutes() == budget_minutes - hosted_eval.COLLECT_RESERVE_MINUTES
+ remaining = hosted_eval.wait_deadline() - time.time()
+ assert 0 < remaining <= (budget_minutes - hosted_eval.COLLECT_RESERVE_MINUTES) * 60 + 1
+ monkeypatch.setenv(hosted_eval.JOB_DEADLINE_ENV, str(time.time() - 60))
+ with pytest.raises(RuntimeError, match="budget"):
+ hosted_eval.evaluation_timeout_minutes()
+ monkeypatch.delenv(hosted_eval.JOB_DEADLINE_ENV)
+ with pytest.raises(RuntimeError, match=hosted_eval.JOB_DEADLINE_ENV):
+ hosted_eval.wait_deadline()
+
+
+def test_wait_bounds_each_poll_and_stops_runs_when_one_stalls(tmp_path: Path, monkeypatch) -> None:
+ output = tmp_path / "hosted"
+ output.mkdir()
+ runs = {"base/swebench-verified": {"evaluation_id": "id-1"}}
+ (output / "hosted-runs.json").write_text(json.dumps(runs))
+ timeouts = []
+
+ def fake_run(command, **kwargs):
+ timeouts.append(kwargs.get("timeout"))
+ return json.dumps({"status": "COMPLETED"})
+
+ args = type("Args", (), {"output": str(output)})()
+ monkeypatch.setattr(hosted_eval, "run", fake_run)
+ monkeypatch.setenv(hosted_eval.JOB_DEADLINE_ENV, str(time.time() + 120 * 60))
+ hosted_eval.wait(args)
+ assert timeouts
+ assert all(value and 0 < value <= hosted_eval.POLL_TIMEOUT_SECONDS for value in timeouts)
+
+ stopped = []
+ monkeypatch.setattr(hosted_eval, "stop_started", lambda records: stopped.append(sorted(records)))
+
+ def stalled_poll(command, **kwargs):
+ raise RuntimeError("prime did not answer within 120s")
+
+ monkeypatch.setattr(hosted_eval, "run", stalled_poll)
+ with pytest.raises(RuntimeError, match="did not answer"):
+ hosted_eval.wait(args)
+ assert stopped == [["base/swebench-verified"]]
+
+
+def test_stop_is_best_effort_and_parses_launcher_logs(tmp_path: Path, monkeypatch) -> None:
+ output = tmp_path / "hosted"
+ logs = output / "launch-logs"
+ logs.mkdir(parents=True)
+ log = logs / "base-swebench-verified.log"
+ log.write_text("Evaluation ID: id-1\n")
+ (output / "hosted-runs.json").write_text(json.dumps({"base/swebench-verified": {"log": str(log)}}))
+ stopped = []
+ timeouts = []
+
+ def fake_run(command, **kwargs):
+ stopped.append(command)
+ timeouts.append(kwargs.get("timeout"))
+
+ monkeypatch.setattr(hosted_eval.subprocess, "run", fake_run)
+ args = type("Args", (), {"output": str(output)})()
+ hosted_eval.stop(args)
+ assert stopped == [["prime", "eval", "stop", "id-1"]]
+ assert timeouts == [hosted_eval.STOP_TIMEOUT_SECONDS]
+ missing = tmp_path / "missing"
+ hosted_eval.stop(type("Args", (), {"output": str(missing)})())
+ assert stopped == [["prime", "eval", "stop", "id-1"]]
+
+ def hung_run(command, **kwargs):
+ raise hosted_eval.subprocess.TimeoutExpired(command, kwargs.get("timeout"))
+
+ monkeypatch.setattr(hosted_eval.subprocess, "run", hung_run)
+ hosted_eval.stop(args)
diff --git a/scripts/evals/short_swe/tests/test_short_swe.py b/scripts/evals/short_swe/tests/test_short_swe.py
new file mode 100644
index 0000000000..a93b1c8906
--- /dev/null
+++ b/scripts/evals/short_swe/tests/test_short_swe.py
@@ -0,0 +1,661 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import subprocess
+import sys
+import tomllib
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[4]
+sys.path.insert(0, str(ROOT))
+
+from scripts.evals.short_swe import ( # noqa: E402
+ builder,
+ ci,
+ cleanup,
+ evaluate,
+ offline_swebench_grader,
+ prepare,
+ report,
+ verified_verifier,
+)
+
+EVAL_ROOT = ROOT / "scripts/evals/short_swe"
+
+
+def test_workflow_gates_and_revokes_durable_release_status() -> None:
+ workflow = (ROOT / ".github/workflows/behavioral-evals.yml").read_text()
+ assert "types: [labeled, unlabeled, synchronize, edited, reopened]" in workflow
+ assert "push:" in workflow
+ assert "github.event.label.name == 'pre-release'" in workflow
+ assert "workflow_dispatch" not in workflow
+ assert "invalidate-on-pr-change:" in workflow
+ assert "invalidate-on-base-change:" in workflow
+ assert "Base advanced; reapply pre-release" in workflow
+ assert "request.head_sha === pull.head.sha && request.base_sha === pull.base.sha" in workflow
+ assert "strict_required_status_checks_policy === true" in workflow
+ assert "Behavioral Eval / pre-release approval" in workflow
+ assert "[...requiredContexts].every" in workflow
+ assert "const strictContexts = new Set(" in workflow
+ assert ".filter(rule => rule.type === 'required_status_checks'" in workflow
+ assert ".flatMap(rule => rule.parameters.required_status_checks.map" in workflow
+ assert "contains(github.event.pull_request.labels.*.name, 'pre-release')" in workflow
+ assert "const rules = await github.paginate(" in workflow
+ assert "GET /repos/{owner}/{repo}/rules/branches/{branch}" in workflow
+ assert workflow.index("Mark the requested head pending") < workflow.index("Check out trusted evaluator")
+ assert "Behavioral Eval / pre-release" in workflow
+ assert "statuses: write" in workflow
+ assert "prime-agent-behavioral-skip-{0}" in workflow
+
+
+def test_manifest_is_the_fixed_pinned_suite() -> None:
+ manifest = json.loads((EVAL_ROOT / "short-swe.json").read_text())
+ prepare.validate_manifest(manifest)
+ assert {item["id"]: len(item["tasks"]) for item in manifest["tasksets"]} == {
+ "swebench-verified": 15,
+ "swebench-pro": 8,
+ "scaleswe": 5,
+ }
+ assert manifest["model"] == "internal/glm-5.3-fast"
+ assert manifest["autonomous"] is False
+ root_concurrency = manifest["max_concurrent"] * 6
+ assert manifest["max_concurrent"] == 4
+ assert root_concurrency <= 32
+ assert manifest["backup_model"] == "internal/deepseek-v4.1-flash"
+ tasksets = {item["id"]: item["tasks"] for item in manifest["tasksets"]}
+ verified_repositories = {task.rsplit("-", 1)[0] for task in tasksets["swebench-verified"]}
+ pro_repositories = {task.removeprefix("instance_").split("-", 1)[0] for task in tasksets["swebench-pro"]}
+ assert len(verified_repositories) == 12
+ assert len(pro_repositories) == 8
+
+
+def test_seaborn_parser_ignores_standalone_status_words() -> None:
+ config = {
+ "instance_id": "mwaskom__seaborn-test",
+ "repo": "mwaskom/seaborn",
+ "FAIL_TO_PASS": json.dumps(["tests/test_fix.py::test_fixed"]),
+ "PASS_TO_PASS": json.dumps(["tests/test_old.py::test_still_works"]),
+ }
+ log = "FAILED\nPASSED\nPASSED tests/test_fix.py::test_fixed\nPASSED tests/test_old.py::test_still_works\n"
+ assert offline_swebench_grader.grade(config, log)[config["instance_id"]]["resolved"] is True
+ with pytest.raises(ValueError, match="missing 2 expected"):
+ offline_swebench_grader.grade(config, "FAILED\nPASSED\n")
+
+
+def test_offline_verified_grader_requires_all_expected_tests() -> None:
+ config = {
+ "instance_id": "astropy__astropy-test",
+ "repo": "astropy/astropy",
+ "FAIL_TO_PASS": json.dumps(["tests/test_fix.py::test_fixed"]),
+ "PASS_TO_PASS": json.dumps(["tests/test_old.py::test_still_works"]),
+ }
+ passed = "tests/test_fix.py::test_fixed PASSED\ntests/test_old.py::test_still_works PASSED\n"
+ report = offline_swebench_grader.grade(config, passed)[config["instance_id"]]
+ assert report["resolved"] is True
+ with pytest.raises(ValueError, match="missing 1 expected"):
+ offline_swebench_grader.grade(config, "tests/test_fix.py::test_fixed PASSED\n")
+ forged_append = offline_swebench_grader.grade(
+ config,
+ "FAILED tests/test_fix.py::test_fixed - assertion\n"
+ "PASSED tests/test_fix.py::test_fixed\n"
+ "PASSED tests/test_old.py::test_still_works\n",
+ )
+ assert forged_append[config["instance_id"]]["resolved"] is False
+ skipped = offline_swebench_grader.grade(
+ config,
+ "tests/test_fix.py::test_fixed PASSED\ntests/test_old.py::test_still_works SKIPPED\n",
+ )
+ assert skipped[config["instance_id"]]["resolved"] is False
+ with pytest.raises(ValueError, match="missing 2 expected"):
+ offline_swebench_grader.grade(config, "pytest failed before collecting tests")
+
+
+def test_patch_collection_uses_trusted_base_and_keeps_all_git_states(tmp_path: Path) -> None:
+ repo = tmp_path / "repo"
+ repo.mkdir()
+ subprocess.run(["git", "init", "-q"], cwd=repo, check=True)
+ subprocess.run(["git", "config", "user.email", "[email protected]"], cwd=repo, check=True)
+ subprocess.run(["git", "config", "user.name", "Test"], cwd=repo, check=True)
+ subprocess.run(["git", "config", "commit.gpgsign", "false"], cwd=repo, check=True)
+ for name in ("committed.txt", "staged.txt", "deleted.txt"):
+ (repo / name).write_text("base\n")
+ (repo / "binary.bin").write_bytes(b"base\x00")
+ subprocess.run(["git", "add", "."], cwd=repo, check=True)
+ subprocess.run(["git", "commit", "-qm", "base"], cwd=repo, check=True)
+ base = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=repo, text=True).strip()
+
+ (repo / "committed.txt").write_text("committed\n")
+ subprocess.run(["git", "add", "committed.txt"], cwd=repo, check=True)
+ subprocess.run(["git", "commit", "-qm", "candidate commit"], cwd=repo, check=True)
+ (repo / "staged.txt").write_text("staged\n")
+ subprocess.run(["git", "add", "staged.txt"], cwd=repo, check=True)
+ (repo / "deleted.txt").unlink()
+ (repo / "binary.bin").write_bytes(b"changed\x00binary")
+ (repo / "untracked.txt").write_text("untracked\n")
+
+ task = tmp_path / "task/tests"
+ task.mkdir(parents=True)
+ (task / "config.json").write_text(json.dumps({"base_commit": base}))
+ command = verified_verifier.patch_collect_command(task.parent)
+ assert command.startswith("rm -rf /logs/artifacts && ")
+ subprocess.run(command.split(" && ", 1)[1], cwd=repo, check=True, shell=True)
+ patch = Path("/tmp/prime-agent.patch")
+ clone = tmp_path / "clone"
+ subprocess.run(["git", "clone", "-q", str(repo), str(clone)], check=True)
+ subprocess.run(["git", "checkout", "-q", base], cwd=clone, check=True)
+ subprocess.run(["git", "apply", "--binary", str(patch)], cwd=clone, check=True)
+ assert (clone / "committed.txt").read_text() == "committed\n"
+ assert (clone / "staged.txt").read_text() == "staged\n"
+ assert not (clone / "deleted.txt").exists()
+ assert (clone / "binary.bin").read_bytes() == b"changed\x00binary"
+ assert (clone / "untracked.txt").read_text() == "untracked\n"
+
+
+def test_verified_test_rewrite_handles_pinned_install_variants() -> None:
+ for install in [*sorted(verified_verifier.INSTALLS), None]:
+ script = "\n".join(
+ filter(
+ None,
+ [
+ install,
+ verified_verifier.LOG_ASSIGNMENT,
+ verified_verifier.TEE_REDIRECT,
+ "pytest tests || true",
+ verified_verifier.PARSER,
+ ],
+ )
+ )
+ rewritten = verified_verifier.rewrite_test_script(script)
+ assert "uv run parser.py" not in rewritten
+ assert "TEST_STATUS=$?" in rewritten
+ assert 'exit "${TEST_STATUS:-0}"' in rewritten
+ assert "/logs/verifier/test.log" not in rewritten
+ assert "output captured by the runtime controller" in rewritten
+ if install:
+ assert "dependencies are pinned in the task image" in rewritten
+ assert "pip install" not in rewritten
+ source = (EVAL_ROOT / "secure_harbor.py").read_text()
+ assert "head -c 16000001" in source
+ assert 'runtime.read("/logs/verifier/test.log"' not in source
+ assert '["rm", "-f", "/tests/config.json", "/tmp/tests.tgz"]' in source
+ with pytest.raises(RuntimeError, match="template"):
+ verified_verifier.rewrite_test_script("python -m pip install malicious")
+
+
+def test_oracle_is_network_blocked_and_required_to_resolve() -> None:
+ manifest = json.loads((EVAL_ROOT / "short-swe.json").read_text())
+ config = tomllib.loads(prepare.oracle_config_text(manifest))
+ assert config["env"]["id"] == "secure_harbor"
+ assert config["env"]["agent"]["runtime"]["allow"] == []
+ assert config["env"]["verifier"]["runtime"]["allow"] == []
+ assert config["env"]["timeout"]["finalize"] == 600
+ assert config["env"]["agent"]["timeout"]["scoring"] == 600
+ assert config["env"]["taskset"]["tasks"] == ["astropy__astropy-14096"]
+ assert config["env"]["agent"]["harness"]["id"] == "oracle_harness"
+ trace = fake_trace()
+ trace.task.data.name = "swe-bench/astropy__astropy-14096"
+ episode = SimpleNamespace(traces=[trace], errors=[])
+ evaluate.validate_oracle_episode(episode)
+ trace.rewards = {"solved": SimpleNamespace(score=0.0, weight=1.0)}
+ trace.reward = 0.0
+ with pytest.raises(RuntimeError, match="oracle"):
+ evaluate.validate_oracle_episode(episode)
+
+
+def labeled_event() -> dict:
+ return {
+ "action": "labeled",
+ "label": {"name": "pre-release"},
+ "pull_request": {
+ "number": 2306,
+ "state": "open",
+ "base": {"sha": "b" * 40, "repo": {"full_name": "PrimeIntellect-ai/prime-agent"}},
+ "head": {"sha": "c" * 40, "repo": {"full_name": "contributor/prime-agent"}},
+ },
+ }
+
+
+def test_task_runtime_credentials_are_removed_before_sandbox_creation(tmp_path: Path) -> None:
+ harbor = tmp_path / "verifiers/v1/tasksets/harbor/taskset.py"
+ harbor.parent.mkdir(parents=True)
+ harbor.write_text(
+ " def runtime_env(self) -> dict[str, str]:\n return resolve_env(self.data.env)\n"
+ )
+ prepare.strip_task_runtime_credentials(tmp_path)
+ namespace = {"resolve_env": lambda env: dict(env)}
+ exec("class Task:\n" + harbor.read_text(), namespace)
+ task = namespace["Task"]()
+ task.data = SimpleNamespace(
+ env={
+ name: "secret"
+ for name in ("PRIME_API_KEY", "PRIME_SANDBOX_API_KEY", "GITHUB_TOKEN", "GH_TOKEN", "HF_TOKEN")
+ }
+ | {"SAFE": "value"}
+ )
+ assert task.runtime_env() == {"SAFE": "value"}
+
+
+def test_ci_resolves_only_exact_label(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ event = tmp_path / "event.json"
+ event.write_text(json.dumps(labeled_event()))
+ monkeypatch.setenv("GITHUB_EVENT_PATH", str(event))
+ monkeypatch.setenv("GITHUB_REPOSITORY", "PrimeIntellect-ai/prime-agent")
+ monkeypatch.setenv("GITHUB_RUN_ID", "7")
+ monkeypatch.setenv("GITHUB_RUN_ATTEMPT", "2")
+ request = ci.resolve(tmp_path / "request", "b" * 40)
+ assert request["base_sha"] == "b" * 40
+ assert request["head_sha"] == "c" * 40
+ with pytest.raises(ValueError, match="identity"):
+ ci.resolve(tmp_path / "stale", "a" * 40)
+ changed = labeled_event()
+ changed["label"]["name"] = "release"
+ event.write_text(json.dumps(changed))
+ with pytest.raises(ValueError, match="pre-release"):
+ ci.resolve(tmp_path / "other", "b" * 40)
+
+
+def fake_trace(*, ok: bool = True, timeout: bool = False):
+ usage = SimpleNamespace(prompt_tokens=11, cached_input_tokens=7, completion_tokens=5)
+ phases = {
+ name: SimpleNamespace(start=1.0, end=2.0, duration=1.0)
+ for name in ("boot", "setup", "agent", "finalize", "scoring")
+ }
+ error = SimpleNamespace(
+ type="HarnessError",
+ message="agent timeout: rollout exceeded its 3600s budget",
+ status_code=None,
+ )
+ call = SimpleNamespace(
+ error=error if timeout else None,
+ node=0,
+ usage=usage,
+ model="internal/glm-5.3-fast",
+ )
+ reward = SimpleNamespace(score=1.0, weight=1.0, value=1.0)
+ return SimpleNamespace(
+ task=SimpleNamespace(data=SimpleNamespace(name="suite/task-1")),
+ rewards={"reward": reward},
+ reward=1.0,
+ ok=ok,
+ is_completed=True,
+ stop_condition=None,
+ errors=[error] if timeout else [],
+ calls=[call],
+ nodes=[SimpleNamespace(parent=None, sampled=True)],
+ usage=usage,
+ timing=SimpleNamespace(**phases),
+ branches=[],
+ )
+
+
+def test_terminal_5xx_tolerates_retried_rate_limits() -> None:
+ outage = SimpleNamespace(type="ProviderError", status_code=503, message="")
+ rate_limit = SimpleNamespace(type="ProviderError", status_code=429, message="rate limited")
+ trace = fake_trace(ok=False, timeout=True)
+ trace.rewards = {}
+ trace.reward = 0.0
+ trace.errors = [outage]
+ trace.calls = [
+ SimpleNamespace(error=rate_limit, node=0, usage=None, model="internal/glm-5.3-fast"),
+ SimpleNamespace(error=outage, node=0, usage=None, model="internal/glm-5.3-fast"),
+ ]
+ record = evaluate.trace_record(SimpleNamespace(traces=[trace], errors=[], ok=False), "suite")
+ assert record["model_failure"] is True
+ trace.calls[0].error = SimpleNamespace(type="ProviderError", status_code=400, message="context limit")
+ with pytest.raises(ValueError, match="complete trace or model outcome"):
+ evaluate.trace_record(SimpleNamespace(traces=[trace], errors=[], ok=False), "suite")
+
+
+def test_non_finite_aggregate_reward_fails() -> None:
+ trace = fake_trace()
+ trace.rewards = {"reward": SimpleNamespace(score=1.0, weight=1.0, value=float("inf"))}
+ trace.reward = float("inf")
+ with pytest.raises(ValueError, match="aggregate reward"):
+ evaluate.trace_record(SimpleNamespace(traces=[trace], errors=[], ok=True), "suite")
+
+
+def test_trace_record_uses_native_usage_buckets() -> None:
+ record = evaluate.trace_record(SimpleNamespace(traces=[fake_trace()], errors=[], ok=True), "suite")
+ assert record["resolved"] is True
+ assert record["uncached_input_tokens"] == 11
+ assert record["cached_input_tokens"] == 7
+ assert record["output_tokens"] == 5
+ assert record["e2e_seconds"] == 5
+ trace = fake_trace()
+ trace.info = {"isolated_verifier_seconds": 3.5}
+ record = evaluate.trace_record(SimpleNamespace(traces=[trace], errors=[], ok=True), "suite")
+ assert record["e2e_seconds"] == 8.5
+
+
+def test_scored_model_timeout_is_an_outcome_but_incomplete_trace_fails() -> None:
+ timeout = fake_trace(ok=False, timeout=True)
+ timeout.rewards = {}
+ timeout.reward = 0.0
+ timeout.calls[0].error = None
+ record = evaluate.trace_record(SimpleNamespace(traces=[timeout], errors=[], ok=False), "suite")
+ assert record["resolved"] is False
+ scored_failure = fake_trace(ok=False, timeout=True)
+ scored_failure.calls[0].error = None
+ with pytest.raises(ValueError, match="complete trace or model outcome"):
+ evaluate.trace_record(SimpleNamespace(traces=[scored_failure], errors=[], ok=False), "suite")
+ scored_failure.rewards["partial"] = None
+ with pytest.raises(ValueError, match="complete trace or model outcome"):
+ evaluate.trace_record(SimpleNamespace(traces=[scored_failure], errors=[], ok=False), "suite")
+ provider_failure = fake_trace(ok=False, timeout=True)
+ provider_failure.rewards = {}
+ provider_failure.reward = 0.0
+ provider_failure.errors = []
+ provider_error = SimpleNamespace(type="ProviderError", status_code=400, message="context limit")
+ provider_failure.calls[0].error = provider_error
+ provider_failure.errors = [provider_error]
+ assert (
+ evaluate.trace_record(SimpleNamespace(traces=[provider_failure], errors=[], ok=False), "suite")[
+ "resolved"
+ ]
+ is False
+ )
+ transient_provider_failure = fake_trace(ok=False, timeout=True)
+ transient_provider_failure.rewards = {}
+ transient_error = SimpleNamespace(type="ProviderError", status_code=429, message="rate limit")
+ transient_provider_failure.calls[0].error = transient_error
+ transient_provider_failure.errors = [transient_error]
+ with pytest.raises(ValueError, match="complete trace or model outcome"):
+ evaluate.trace_record(
+ SimpleNamespace(traces=[transient_provider_failure], errors=[], ok=False), "suite"
+ )
+ mixed_failure = fake_trace(ok=False, timeout=True)
+ mixed_failure.rewards = {}
+ mixed_failure.calls[0].error = None
+ mixed_failure.errors.insert(0, transient_error)
+ with pytest.raises(ValueError, match="complete trace or model outcome"):
+ evaluate.trace_record(SimpleNamespace(traces=[mixed_failure], errors=[], ok=False), "suite")
+ contaminated_timeout = fake_trace(ok=False, timeout=True)
+ contaminated_timeout.rewards = {}
+ contaminated_timeout.calls[0].error = transient_error
+ with pytest.raises(ValueError, match="complete trace or model outcome"):
+ evaluate.trace_record(SimpleNamespace(traces=[contaminated_timeout], errors=[], ok=False), "suite")
+ credential_failure = fake_trace(ok=False, timeout=True)
+ credential_failure.rewards = {}
+ credential_error = SimpleNamespace(type="ProviderError", status_code=401, message="unauthorized")
+ credential_failure.calls[0].error = credential_error
+ credential_failure.errors = [credential_error]
+ with pytest.raises(ValueError, match="complete trace or model outcome"):
+ evaluate.trace_record(SimpleNamespace(traces=[credential_failure], errors=[], ok=False), "suite")
+ unrelated_terminal = fake_trace(ok=False, timeout=True)
+ unrelated_terminal.rewards = {}
+ unrelated_terminal.calls[0].error = provider_error
+ unrelated_terminal.errors = [SimpleNamespace(type="SandboxError", status_code=None, message="lost")]
+ with pytest.raises(ValueError, match="complete trace or model outcome"):
+ evaluate.trace_record(SimpleNamespace(traces=[unrelated_terminal], errors=[], ok=False), "suite")
+ infrastructure = fake_trace(ok=False)
+ infrastructure.rewards = {}
+ infrastructure.reward = 0.0
+ infrastructure.errors = [SimpleNamespace(type="HarnessError", message="sandbox unavailable")]
+ with pytest.raises(ValueError, match="complete trace or model outcome"):
+ evaluate.trace_record(SimpleNamespace(traces=[infrastructure], errors=[], ok=False), "suite")
+ incomplete = fake_trace()
+ incomplete.is_completed = False
+ incomplete.ok = False
+ with pytest.raises(ValueError, match="complete trace or model outcome"):
+ evaluate.trace_record(SimpleNamespace(traces=[incomplete], errors=[], ok=False), "suite")
+
+
+def test_model_failure_without_reported_usage_records_zero_tokens() -> None:
+ failure = fake_trace(ok=False, timeout=True)
+ failure.rewards = {}
+ failure.reward = 0.0
+ failure.usage = None
+ provider_error = SimpleNamespace(type="ProviderError", status_code=503, message="connection reset")
+ failure.calls[0].error = provider_error
+ failure.calls[0].usage = None
+ failure.errors = [provider_error]
+ record = evaluate.trace_record(SimpleNamespace(traces=[failure], errors=[], ok=False), "suite")
+ assert record["model_failure"] is True
+ assert (record["uncached_input_tokens"], record["cached_input_tokens"], record["output_tokens"]) == (
+ 0,
+ 0,
+ 0,
+ )
+ scored = fake_trace()
+ scored.usage = None
+ scored.calls[0].usage = None
+ with pytest.raises(ValueError, match="incomplete provider usage"):
+ evaluate.trace_record(SimpleNamespace(traces=[scored], errors=[], ok=True), "suite")
+
+
+ ("start", "end"),
+ ((float("nan"), 2.0), (1.0, float("inf")), (3.0, 2.0), (0.0, 2.0)),
+)
+def test_trace_timing_rejects_nonfinite_reversed_or_partial_spans(start: float, end: float) -> None:
+ trace = fake_trace()
+ trace.timing.agent.start = start
+ trace.timing.agent.end = end
+ with pytest.raises(ValueError, match="invalid timing"):
+ evaluate.validate_timing(trace)
+
+
+def test_trace_graph_allows_uncommitted_successful_call() -> None:
+ trace = fake_trace()
+ trace.calls[0].node = None
+ trace.calls[0].error = None
+ evaluate.validate_graph(trace)
+
+
+ ("parent", "call_node"),
+ ((1, 1), (1, -1), (2, 0)),
+)
+def test_trace_graph_rejects_invalid_links(parent: int | None, call_node: int) -> None:
+ trace = fake_trace()
+ trace.nodes[0].parent = parent
+ trace.calls[0].node = call_node
+ with pytest.raises(ValueError, match="invalid"):
+ evaluate.validate_graph(trace)
+
+
+def tasks(resolved: int, multiplier: int = 1) -> list[dict]:
+ rows = []
+ names = [("swebench-verified", 15), ("swebench-pro", 8), ("scaleswe", 5)]
+ index = 0
+ for taskset, count in names:
+ for _local in range(count):
+ rows.append(
+ {
+ "taskset": taskset,
+ "task": f"task-{index}",
+ "resolved": index < resolved,
+ "model_failure": False,
+ "uncached_input_tokens": 100 * multiplier,
+ "cached_input_tokens": 200 * multiplier,
+ "output_tokens": 50 * multiplier,
+ "e2e_seconds": float(10 * multiplier),
+ "model_calls": 2,
+ }
+ )
+ index += 1
+ return rows
+
+
+def request() -> dict:
+ return {
+ "schema_version": 1,
+ "repository": "PrimeIntellect-ai/prime-agent",
+ "head_repository": "PrimeIntellect-ai/prime-agent",
+ "pr": 2306,
+ "run_id": 7,
+ "attempt": 1,
+ "harness_sha": "a" * 40,
+ "base_sha": "b" * 40,
+ "head_sha": "c" * 40,
+ }
+
+
+def paired(base_resolved: int = 10, head_resolved: int = 10, head_multiplier: int = 1) -> dict:
+ return {
+ "schema_version": 1,
+ "request": request(),
+ "model": "internal/glm-5.3-fast",
+ "started_at": 1.0,
+ "finished_at": 2.0,
+ "sides": {"base": tasks(base_resolved), "head": tasks(head_resolved, head_multiplier)},
+ }
+
+
+def test_report_passes_noise_and_colors_meaningful_token_change() -> None:
+ result = paired(head_resolved=11, head_multiplier=2)
+ markdown, verdict = report.render(result, request())
+ assert verdict == "pass"
+ assert "\\textcolor" in markdown
+ assert "#e5484d" in markdown
+ assert "SWE-bench Verified" in markdown
+ assert "Cumulative task time | 280.0 s | 560.0 s" in markdown
+ assert "concurrent tasks overlap in wall-clock time" in markdown
+ assert "End-to-end time" not in markdown
+ invalid = paired()
+ invalid["sides"]["head"][0]["e2e_seconds"] = -1.0
+ with pytest.raises(ValueError, match="e2e_seconds"):
+ report.render(invalid, request())
+ contradictory = paired(head_resolved=1)
+ contradictory["sides"]["head"][0]["model_failure"] = True
+ with pytest.raises(ValueError, match="outcome"):
+ report.render(contradictory, request())
+ assert "task-0" not in markdown
+
+
+def test_report_fails_drastic_quality_or_efficiency_regression() -> None:
+ quality = paired(base_resolved=10, head_resolved=5)
+ _, verdict = report.render(quality, request())
+ assert verdict == "fail"
+ failures = paired()
+ for task in failures["sides"]["head"][:3]:
+ task["resolved"] = False
+ task["model_failure"] = True
+ markdown, verdict = report.render(failures, request())
+ assert verdict == "fail"
+ assert "Model failures increased by 3" in markdown
+ assert "#e5484d" in markdown
+ efficiency = paired(base_resolved=10, head_resolved=10, head_multiplier=2)
+ markdown, verdict = report.render(efficiency, request())
+ assert verdict == "fail"
+ assert "Cumulative task time reached 2.00x base without more resolutions." in markdown
+
+
+def open_directories(source: Path, destination: Path) -> tuple[int, int]:
+ flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC
+ return os.open(source, flags), os.open(destination, flags)
+
+
+def test_cleanup_rechecks_labels_paginates_and_attempts_every_delete() -> None:
+ labels = ["owner", "repository:repo/name", "run:1", "attempt:1"]
+ pages = [
+ SimpleNamespace(
+ sandboxes=[
+ SimpleNamespace(id="owned-1", labels=[*labels, "role:task"]),
+ SimpleNamespace(id="foreign", labels=["owner"]),
+ ],
+ has_next=True,
+ ),
+ SimpleNamespace(
+ sandboxes=[SimpleNamespace(id="owned-2", labels=labels)],
+ has_next=False,
+ ),
+ ]
+
+ class Client:
+ def __init__(self):
+ self.deleted = []
+
+ def list(self, **kwargs):
+ return pages[kwargs["page"] - 1]
+
+ def delete(self, sandbox_id):
+ self.deleted.append(sandbox_id)
+ if sandbox_id == "owned-1":
+ raise RuntimeError("failed")
+
+ client = Client()
+ with pytest.raises(RuntimeError, match="1 sandbox"):
+ cleanup.cleanup_owned(client, labels)
+ assert client.deleted == ["owned-1", "owned-2"]
+
+
+def test_artifact_snapshot_hashes_one_open_regular_file(tmp_path: Path) -> None:
+ source, destination = tmp_path / "source", tmp_path / "destination"
+ source.mkdir()
+ destination.mkdir()
+ original = b"trusted snapshot bytes"
+ (source / "artifact.tgz").write_bytes(original)
+ source_fd, destination_fd = open_directories(source, destination)
+ try:
+ record = builder.snapshot_artifact(source_fd, destination_fd, "artifact.tgz")
+ finally:
+ os.close(source_fd)
+ os.close(destination_fd)
+ (source / "artifact.tgz").write_bytes(b"replacement")
+ assert (destination / "artifact.tgz").read_bytes() == original
+ assert record == {
+ "name": "artifact.tgz",
+ "size": len(original),
+ "sha256": hashlib.sha256(original).hexdigest(),
+ }
+
+
+def test_artifact_snapshot_rejects_symlink_fifo_and_oversize(tmp_path: Path) -> None:
+ source, destination = tmp_path / "source", tmp_path / "destination"
+ source.mkdir()
+ destination.mkdir()
+ (source / "regular").write_bytes(b"data")
+ (source / "symlink.tgz").symlink_to("regular")
+ os.mkfifo(source / "fifo.tgz")
+ with (source / "large.tgz").open("wb") as stream:
+ stream.truncate(builder.MAX_ARTIFACT_BYTES + 1)
+ source_fd, destination_fd = open_directories(source, destination)
+ try:
+ for name in ("symlink.tgz", "fifo.tgz", "large.tgz"):
+ with pytest.raises(ValueError):
+ builder.snapshot_artifact(source_fd, destination_fd, name)
+ finally:
+ os.close(source_fd)
+ os.close(destination_fd)
+ assert list(destination.iterdir()) == []
+
+
+def test_artifact_directory_rejects_intermediate_symlink(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ source, outside = tmp_path / "source", tmp_path / "outside"
+ source.mkdir()
+ outside.mkdir()
+ (outside / "artifacts").mkdir()
+ (source / "linked").symlink_to(outside, target_is_directory=True)
+ monkeypatch.setattr(builder, "SOURCE", source)
+ monkeypatch.setattr(builder, "ARTIFACT_RELATIVE", Path("linked/artifacts"))
+ with pytest.raises(OSError):
+ builder.open_candidate_artifacts()
+
+
+def test_compare_gates_positive_totals_over_a_zero_base() -> None:
+ base = {
+ "resolved": 5,
+ "model_failures": 0,
+ "uncached_input_tokens": 1,
+ "cached_input_tokens": 1,
+ "output_tokens": 0,
+ "e2e_seconds": 0.0,
+ }
+ head = {
+ "resolved": 5,
+ "model_failures": 0,
+ "uncached_input_tokens": 1,
+ "cached_input_tokens": 1,
+ "output_tokens": 100,
+ "e2e_seconds": 10.0,
+ }
+ findings = report.compare(base, head)
+ assert findings, "positive totals over a zero base must gate"
diff --git a/scripts/evals/short_swe/verified_verifier.py b/scripts/evals/short_swe/verified_verifier.py
new file mode 100644
index 0000000000..40d79f95ec
--- /dev/null
+++ b/scripts/evals/short_swe/verified_verifier.py
@@ -0,0 +1,54 @@
+"""Trusted rewrites for pinned SWE-bench Verified test templates."""
+
+import json
+from pathlib import Path
+
+INSTALLS = {
+ "python -m pip install -e .[test] --verbose",
+ "python -m pip install -e .",
+ "python -m pip install -e .[dev]",
+ "python -m pip install -e .[test]",
+ "python -m pip install .",
+}
+PARSER = 'uv run parser.py | tee -a "$LOG_FILE"'
+LOG_ASSIGNMENT = "LOG_FILE=$(mktemp)"
+TEE_REDIRECT = 'exec > >(tee "$LOG_FILE") 2>&1'
+
+
+def trusted_base_commit(task_dir: Path) -> str:
+ config = json.loads((task_dir / "tests" / "config.json").read_text())
+ base = config.get("base_commit")
+ if not isinstance(base, str) or len(base) != 40 or any(ch not in "0123456789abcdef" for ch in base):
+ raise ValueError("invalid SWE-bench base commit")
+ return base
+
+
+def patch_collect_command(task_dir: Path) -> str:
+ base = trusted_base_commit(task_dir)
+ return (
+ "rm -rf /logs/artifacts && "
+ "git add -N -- . && "
+ f"git diff --binary --no-ext-diff {base} -- . > /tmp/prime-agent.patch"
+ )
+
+
+def rewrite_test_script(script: str) -> str:
+ install_lines = [line for line in script.splitlines() if line.strip().startswith("python -m pip install")]
+ if (
+ script.count(PARSER) != 1
+ or script.count(LOG_ASSIGNMENT) != 1
+ or script.count(TEE_REDIRECT) != 1
+ or script.count(" || true") != 1
+ or len(install_lines) > 1
+ or any(line.strip() not in INSTALLS for line in install_lines)
+ ):
+ raise RuntimeError("SWE-bench verifier template did not match")
+ for line in install_lines:
+ replacement = line[: len(line) - len(line.lstrip())] + (
+ ": # dependencies are pinned in the task image; test the mounted source tree"
+ )
+ script = script.replace(line, replacement, 1)
+ script = script.replace(" || true", " || TEST_STATUS=$?", 1)
+ script = script.replace(LOG_ASSIGNMENT, "LOG_FILE=/dev/null", 1)
+ script = script.replace(TEE_REDIRECT, ": # output captured by the runtime controller", 1)
+ return script.replace(PARSER, 'exit "${TEST_STATUS:-0}"', 1)