The Open Framework for Building Agents
ci(eve-code): run /benchmark on a chosen dataset (#4019)
Ruia7b0fb2bf24d
diff --git a/.github/scripts/resolve-benchmark-request.mjs b/.github/scripts/resolve-benchmark-request.mjsindex dfd02b159..e6c2465de 100644--- a/.github/scripts/resolve-benchmark-request.mjs+++ b/.github/scripts/resolve-benchmark-request.mjs@@ -11,8 +11,33 @@ export const BENCHMARK_HARNESSES = [/** Harnesses compared on every automatic run; eve-code against opencode and pi. */export const DEFAULT_HARNESSES = ["eve-code", "opencode", "pi"];-function parseHarnesses(value) {- const harnesses = [...new Set(value.split(/[ ,]+/u).filter(Boolean))];+/**+ * eve-bench datasets a benchmark can run, with the settings each was calibrated for.+ * swe-lean needs five attempts to separate a 10-15% token change from trial noise.+ * deepswe-lean resolves almost nothing at low reasoning, and its 20-minute trials+ * make three attempts the most that fits one job.+ */+export const BENCHMARK_DATASETS = {+ "swe-lean": { reasoning: "low", attempts: "5" },+ "deepswe-lean": { reasoning: "high", attempts: "3" },+};++/** The dataset of every automatic run and of a request that names none. */+export const DEFAULT_DATASET = "swe-lean";++function parseDataset(value) {+ if (!Object.hasOwn(BENCHMARK_DATASETS, value)) {+ throw new Error(`Supported benchmark datasets: ${Object.keys(BENCHMARK_DATASETS).join(", ")}.`);+ }+ return value;+}++function splitNames(value) {+ return value.split(/[ ,]+/u).filter(Boolean);+}++function parseHarnesses(names) {+ const harnesses = [...new Set(names)];const unknown = harnesses.filter((harness) => !BENCHMARK_HARNESSES.includes(harness));if (!harnesses.length || unknown.length) {throw new Error(`Supported benchmark harnesses: ${BENCHMARK_HARNESSES.join(", ")}.`);@@ -20,13 +45,32 @@ function parseHarnesses(value) {return harnesses;}-function request(harnesses, sha, pr) {- return { harness: harnesses.join(","), slug: harnesses.join("+"), sha, pr };+/** `[<dataset>] [<harness>...]`, in any order; a dataset or the harness list may be omitted. */+function parseSelection(value) {+ const names = splitNames(value);+ const datasets = names.filter((name) => Object.hasOwn(BENCHMARK_DATASETS, name));+ if (datasets.length > 1) throw new Error("Name at most one benchmark dataset.");+ const harnesses = names.filter((name) => !datasets.includes(name));+ return {+ dataset: datasets[0] ?? DEFAULT_DATASET,+ harnesses: harnesses.length ? parseHarnesses(harnesses) : DEFAULT_HARNESSES,+ };+}++function request({ dataset, harnesses }, sha, pr) {+ return {+ harness: harnesses.join(","),+ slug: harnesses.join("+"),+ dataset,+ ...BENCHMARK_DATASETS[dataset],+ sha,+ pr,+ };}export async function resolveBenchmarkRequest({ github, context }) {const { payload, eventName, repo, sha } = context;- let harnesses = DEFAULT_HARNESSES;+ let selection = { dataset: DEFAULT_DATASET, harnesses: DEFAULT_HARNESSES };let number = payload.pull_request?.number;if (eventName === "issue_comment") {@@ -39,18 +83,23 @@ export async function resolveBenchmarkRequest({ github, context }) {});if (!["admin", "maintain", "write"].includes(data.permission)) return null;const command = /^\/benchmark(?: +([a-z-]+(?:[ ,]+[a-z-]+)*))?$/u.exec(body);- if (!command) throw new Error("Use /benchmark or /benchmark <harness>[,<harness>...].");- if (command[1]) harnesses = parseHarnesses(command[1]);+ if (!command) throw new Error("Use /benchmark [<dataset>] [<harness>[,<harness>...]].");+ if (command[1]) selection = parseSelection(command[1]);number = payload.issue.number;} else if (eventName === "workflow_dispatch") {- if (payload.inputs?.harness) harnesses = parseHarnesses(payload.inputs.harness);+ selection = {+ dataset: parseDataset(payload.inputs?.dataset || DEFAULT_DATASET),+ harnesses: payload.inputs?.harness+ ? parseHarnesses(splitNames(payload.inputs.harness))+ : DEFAULT_HARNESSES,+ };} else if (eventName !== "pull_request" && eventName !== "push") {return null;}if (!number) {if (!/^[a-f0-9]{40}$/u.test(sha)) throw new Error("Benchmark source must be an exact commit.");- return request(harnesses, sha, "");+ return request(selection, sha, "");}const { data: pull } = await github.rest.pulls.get({ ...repo, pull_number: number });@@ -64,5 +113,5 @@ export async function resolveBenchmarkRequest({ github, context }) {return null;if (eventName === "pull_request" && pull.head.sha !== payload.pull_request.head.sha) return null;if (!/^[a-f0-9]{40}$/u.test(pull.head.sha)) throw new Error("PR head must be an exact commit.");- return request(harnesses, pull.head.sha, String(number));+ return request(selection, pull.head.sha, String(number));}diff --git a/.github/scripts/resolve-benchmark-request.test.mjs b/.github/scripts/resolve-benchmark-request.test.mjsindex 823725f77..3fe444206 100644--- a/.github/scripts/resolve-benchmark-request.test.mjs+++ b/.github/scripts/resolve-benchmark-request.test.mjs@@ -3,12 +3,16 @@ import { readFile } from "node:fs/promises";import test from "node:test";import {+ BENCHMARK_DATASETS,BENCHMARK_HARNESSES,+ DEFAULT_DATASET,DEFAULT_HARNESSES,resolveBenchmarkRequest,} from "./resolve-benchmark-request.mjs";-const defaults = { harness: "eve-code,opencode,pi", slug: "eve-code+opencode+pi" };+const sweLean = { dataset: "swe-lean", reasoning: "low", attempts: "5" };+const deepsweLean = { dataset: "deepswe-lean", reasoning: "high", attempts: "3" };+const defaults = { harness: "eve-code,opencode,pi", slug: "eve-code+opencode+pi", ...sweLean };const sha = "a".repeat(40);const pull = {@@ -82,6 +86,7 @@ for (const harness of BENCHMARK_HARNESSES) {assert.deepEqual(await resolve(fixture({ body: `/benchmark ${harness}` })), {harness,slug: harness,+ ...sweLean,sha,pr: "42",});@@ -100,6 +105,45 @@ test("a comment can select several harnesses, comma- or space-separated, dedupli}});+test("manual dispatch offers exactly the datasets the resolver accepts", async () => {+ const workflow = await readFile(+ new URL("../workflows/eve-code-benchmark.yml", import.meta.url),+ "utf8",+ );+ const input = /\n {6}dataset:\n(?: {8}.*\n)+/u.exec(workflow)?.[0] ?? "";+ const options = /^ {8}options: \[(.*)\]$/mu.exec(input)?.[1].split(/, */u);+ assert.deepEqual(options, Object.keys(BENCHMARK_DATASETS));+ assert.equal(/^ {8}default: (\S+)$/mu.exec(input)?.[1], DEFAULT_DATASET);+});++test("a comment can select a dataset, with the default harnesses or its own list", async () => {+ assert.deepEqual(await resolve(fixture({ body: "/benchmark deepswe-lean" })), {+ ...defaults,+ ...deepsweLean,+ sha,+ pr: "42",+ });+ for (const body of [+ "/benchmark deepswe-lean eve-code,pi",+ "/benchmark eve-code pi deepswe-lean",+ "/benchmark eve-code, deepswe-lean, pi",+ ]) {+ const result = await resolve(fixture({ body }));+ assert.deepEqual(+ [result.dataset, result.reasoning, result.attempts, result.harness, result.slug],+ ["deepswe-lean", "high", "3", "eve-code,pi", "eve-code+pi"],+ );+ }+ assert.equal((await resolve(fixture({ body: "/benchmark swe-lean pi" }))).dataset, "swe-lean");+});++test("a comment cannot name two datasets", async () => {+ await assert.rejects(+ resolve(fixture({ body: "/benchmark swe-lean deepswe-lean" })),+ /at most one benchmark dataset/u,+ );+});+for (const permission of ["read", "triage", "none"]) {test(`${permission} permission cannot launch a benchmark`, async () => {const input = fixture({ permission });@@ -204,6 +248,19 @@ test("manual dispatch defaults to the default harnesses and validates an expliciawait assert.rejects(resolve(input), /Supported benchmark harnesses/u);});+test("manual dispatch can publish a deepswe-lean result and rejects unknown datasets", async () => {+ const input = fixture({ eventName: "workflow_dispatch" });+ input.context.payload.inputs = { harness: "", dataset: "deepswe-lean" };+ assert.deepEqual(await resolve(input), {+ ...defaults,+ ...deepsweLean,+ sha: input.context.sha,+ pr: "",+ });+ input.context.payload.inputs.dataset = "terminal-bench";+ await assert.rejects(resolve(input), /Supported benchmark datasets: swe-lean, deepswe-lean/u);+});+test("pushes to main publish the default harnesses at the pushed commit", async () => {const input = fixture({ eventName: "push" });assert.deepEqual(await resolve(input), { ...defaults, sha: input.context.sha, pr: "" });@@ -226,14 +283,18 @@ test("workflow names and concurrency preserve independent authorized harness runworkflow,/needs: request\n\s+if: needs\.request\.outputs\.run == 'true'\n\s+concurrency:/u,);+ // A deepswe-lean run never cancels or overwrites a swe-lean run of the same harnesses.+ assert.match(+ workflow,+ /group: eve-code-.*needs\.request\.outputs\.pr.*needs\.request\.outputs\.dataset.*needs\.request\.outputs\.slug/u,+ );assert.match(workflow,- /group: eve-code-.*needs\.request\.outputs\.pr.*needs\.request\.outputs\.slug/u,+ /artifact-name: \$\{\{ needs\.request\.outputs\.dataset \}\}-\$\{\{ needs\.request\.outputs\.slug \}\}/u,);- assert.match(workflow, /artifact-name: swe-lean-\$\{\{ needs\.request\.outputs\.slug \}\}/u);assert.match(workflow, /push:\n\s+branches: \[main\]/u);- // Five attempts per task, run in waves of the action's maximum of 16 sandboxes.- assert.match(workflow, /attempts: "5"/u);+ // Each dataset's attempts run in waves of the action's maximum of 16 sandboxes.+ assert.match(workflow, /attempts: \$\{\{ needs\.request\.outputs\.attempts \}\}/u);assert.match(workflow, /concurrency: "16"/u);assert.match(workflow, /agent-ref: \$\{\{ needs\.request\.outputs\.sha \}\}/u);});@@ -249,11 +310,11 @@ test("the consumer tracks eve-bench main and owns its model selection", async ()workflow,/model: \$\{\{ vars\.EVE_CODE_BENCH_MODEL \|\| 'anthropic\/claude-sonnet-5\.5' \}\}/u,);- assert.match(workflow, /reasoning: low\n/u);+ assert.match(workflow, /reasoning: \$\{\{ needs\.request\.outputs\.reasoning \}\}\n/u);assert.doesNotMatch(workflow, /runner-revision:/u);assert.match(workflow, /blob-token: \$\{\{ secrets\.EVE_BENCH_BLOB_READ_WRITE_TOKEN \}\}/u);assert.doesNotMatch(workflow, /artifact-id/u);- assert.match(workflow, /dataset: swe-lean\n/u);+ assert.match(workflow, /dataset: \$\{\{ needs\.request\.outputs\.dataset \}\}\n/u);assert.match(workflow, /contenders: eve-code@baseline,eve-code@head\n/u);assert.doesNotMatch(workflow, /^\s+task:/mu);assert.equal((workflow.match(/uses: \.\/\.eve-bench-action/gu) ?? []).length, 1);diff --git a/.github/workflows/eve-code-benchmark.yml b/.github/workflows/eve-code-benchmark.ymlindex 1cae8b964..d844cc497 100644--- a/.github/workflows/eve-code-benchmark.yml+++ b/.github/workflows/eve-code-benchmark.yml@@ -6,8 +6,10 @@ on:- ".github/workflows/eve-code-benchmark.yml"- ".github/scripts/resolve-benchmark-request*"- "packages/eve-code/**"- # Every eve-code change on main publishes a fresh result per harness, so PRs- # compare their HEAD against the latest main as the baseline.+ # Every eve-code change on main publishes a fresh swe-lean result per harness,+ # so PRs compare their HEAD against the latest main as the baseline. Pushes do+ # not run deepswe-lean: its baseline is only as fresh as the last manual+ # dispatch, so a PR's delta there also includes eve-code changes merged since.push:branches: [main]paths:@@ -17,9 +19,14 @@ on:workflow_dispatch:inputs:harness:- description: Comma-separated harnesses to benchmark on SWE-lean+ description: Comma-separated harnesses to benchmarktype: stringdefault: eve-code,opencode,pi+ dataset:+ description: eve-bench dataset to run+ type: choice+ options: [swe-lean, deepswe-lean]+ default: swe-lean# OIDC alternative: add `id-token: write` here and omit `vercel-token` below# after configuring a matching Vercel team OIDC policy.@@ -42,6 +49,9 @@ jobs:run: ${{ steps.request.outputs.run }}harness: ${{ steps.request.outputs.harness }}slug: ${{ steps.request.outputs.slug }}+ dataset: ${{ steps.request.outputs.dataset }}+ reasoning: ${{ steps.request.outputs.reasoning }}+ attempts: ${{ steps.request.outputs.attempts }}sha: ${{ steps.request.outputs.sha }}pr: ${{ steps.request.outputs.pr }}steps:@@ -64,12 +74,13 @@ jobs:needs: requestif: needs.request.outputs.run == 'true'concurrency:- group: eve-code-${{ needs.request.outputs.pr || github.ref }}-${{ needs.request.outputs.slug }}+ group: eve-code-${{ needs.request.outputs.pr || github.ref }}-${{ needs.request.outputs.dataset }}-${{ needs.request.outputs.slug }}cancel-in-progress: trueruns-on: ubuntu-latest- # Trials run in waves of up to 16 sandboxes; each trial can take up to the- # 15-minute agent timeout plus preparation and verification.- timeout-minutes: 90+ # Trials run in waves of up to 16 sandboxes; each trial can take up to its+ # dataset's agent timeout (20 minutes on deepswe-lean) plus preparation and+ # verification.+ timeout-minutes: 120steps:# vercel/eve is public and eve-bench is private; GitHub never lets a public# repository `uses:` a private action, so fetch main with a read-only deploy@@ -89,24 +100,24 @@ jobs:agent-repository: vercel/eveagent-paths: '["packages/eve-code"]'agent-ref: ${{ needs.request.outputs.sha }}- dataset: swe-lean+ dataset: ${{ needs.request.outputs.dataset }}# The PR comment's ranking tables show these plus each dimension's winner.contenders: eve-code@baseline,eve-code@head# Sonnet keeps Anthropic's prompt-cache behavior (tools first, cache writes)# visible at half Opus's price. Avoid Gemini: opencode drops its tool-call# thought signatures, so it stops reasoning after the first steps there.model: ${{ vars.EVE_CODE_BENCH_MODEL || 'anthropic/claude-sonnet-5.5' }}- reasoning: low- # One attempt per task could not separate a 10-15% token change from- # trial noise; five gives the paired bootstrap enough samples.- attempts: "5"- # The action's maximum. 7 tasks x 5 attempts per contender run in waves.+ # Reasoning and attempts are calibrated per dataset; see+ # .github/scripts/resolve-benchmark-request.mjs.+ reasoning: ${{ needs.request.outputs.reasoning }}+ attempts: ${{ needs.request.outputs.attempts }}+ # The action's maximum; each contender's tasks x attempts run in waves.concurrency: "16"native-build: "false"# Baselines and cross-harness references resolve automatically from the# durable result store; only dispatch/push/schedule runs publish to it.blob-token: ${{ secrets.EVE_BENCH_BLOB_READ_WRITE_TOKEN }}- artifact-name: swe-lean-${{ needs.request.outputs.slug }}+ artifact-name: ${{ needs.request.outputs.dataset }}-${{ needs.request.outputs.slug }}gateway-key: ${{ secrets.AI_GATEWAY_API_KEY }}github-token: ${{ github.token }}vercel-token: ${{ secrets.VERCEL_TOKEN }}