Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

ci(eve-code): run /benchmark on a chosen dataset (#4019)

Ruia7b0fb2bf24d
diff --git a/.github/scripts/resolve-benchmark-request.mjs b/.github/scripts/resolve-benchmark-request.mjs
index dfd02b159..e6c2465de 100644
--- a/.github/scripts/resolve-benchmark-request.mjs
+++ b/.github/scripts/resolve-benchmark-request.mjs
@@ -11,8 +11,33 @@ export const BENCHMARK_HARNESSES = [
/** Harnesses compared on every automatic run; eve-code against opencode and pi. */
export const DEFAULT_HARNESSES = ["eve-code", "opencode", "pi"];
-function parseHarnesses(value) {
- const harnesses = [...new Set(value.split(/[ ,]+/u).filter(Boolean))];
+/**
+ * eve-bench datasets a benchmark can run, with the settings each was calibrated for.
+ * swe-lean needs five attempts to separate a 10-15% token change from trial noise.
+ * deepswe-lean resolves almost nothing at low reasoning, and its 20-minute trials
+ * make three attempts the most that fits one job.
+ */
+export const BENCHMARK_DATASETS = {
+ "swe-lean": { reasoning: "low", attempts: "5" },
+ "deepswe-lean": { reasoning: "high", attempts: "3" },
+};
+
+/** The dataset of every automatic run and of a request that names none. */
+export const DEFAULT_DATASET = "swe-lean";
+
+function parseDataset(value) {
+ if (!Object.hasOwn(BENCHMARK_DATASETS, value)) {
+ throw new Error(`Supported benchmark datasets: ${Object.keys(BENCHMARK_DATASETS).join(", ")}.`);
+ }
+ return value;
+}
+
+function splitNames(value) {
+ return value.split(/[ ,]+/u).filter(Boolean);
+}
+
+function parseHarnesses(names) {
+ const harnesses = [...new Set(names)];
const unknown = harnesses.filter((harness) => !BENCHMARK_HARNESSES.includes(harness));
if (!harnesses.length || unknown.length) {
throw new Error(`Supported benchmark harnesses: ${BENCHMARK_HARNESSES.join(", ")}.`);
@@ -20,13 +45,32 @@ function parseHarnesses(value) {
return harnesses;
}
-function request(harnesses, sha, pr) {
- return { harness: harnesses.join(","), slug: harnesses.join("+"), sha, pr };
+/** `[<dataset>] [<harness>...]`, in any order; a dataset or the harness list may be omitted. */
+function parseSelection(value) {
+ const names = splitNames(value);
+ const datasets = names.filter((name) => Object.hasOwn(BENCHMARK_DATASETS, name));
+ if (datasets.length > 1) throw new Error("Name at most one benchmark dataset.");
+ const harnesses = names.filter((name) => !datasets.includes(name));
+ return {
+ dataset: datasets[0] ?? DEFAULT_DATASET,
+ harnesses: harnesses.length ? parseHarnesses(harnesses) : DEFAULT_HARNESSES,
+ };
+}
+
+function request({ dataset, harnesses }, sha, pr) {
+ return {
+ harness: harnesses.join(","),
+ slug: harnesses.join("+"),
+ dataset,
+ ...BENCHMARK_DATASETS[dataset],
+ sha,
+ pr,
+ };
}
export async function resolveBenchmarkRequest({ github, context }) {
const { payload, eventName, repo, sha } = context;
- let harnesses = DEFAULT_HARNESSES;
+ let selection = { dataset: DEFAULT_DATASET, harnesses: DEFAULT_HARNESSES };
let number = payload.pull_request?.number;
if (eventName === "issue_comment") {
@@ -39,18 +83,23 @@ export async function resolveBenchmarkRequest({ github, context }) {
});
if (!["admin", "maintain", "write"].includes(data.permission)) return null;
const command = /^\/benchmark(?: +([a-z-]+(?:[ ,]+[a-z-]+)*))?$/u.exec(body);
- if (!command) throw new Error("Use /benchmark or /benchmark <harness>[,<harness>...].");
- if (command[1]) harnesses = parseHarnesses(command[1]);
+ if (!command) throw new Error("Use /benchmark [<dataset>] [<harness>[,<harness>...]].");
+ if (command[1]) selection = parseSelection(command[1]);
number = payload.issue.number;
} else if (eventName === "workflow_dispatch") {
- if (payload.inputs?.harness) harnesses = parseHarnesses(payload.inputs.harness);
+ selection = {
+ dataset: parseDataset(payload.inputs?.dataset || DEFAULT_DATASET),
+ harnesses: payload.inputs?.harness
+ ? parseHarnesses(splitNames(payload.inputs.harness))
+ : DEFAULT_HARNESSES,
+ };
} else if (eventName !== "pull_request" && eventName !== "push") {
return null;
}
if (!number) {
if (!/^[a-f0-9]{40}$/u.test(sha)) throw new Error("Benchmark source must be an exact commit.");
- return request(harnesses, sha, "");
+ return request(selection, sha, "");
}
const { data: pull } = await github.rest.pulls.get({ ...repo, pull_number: number });
@@ -64,5 +113,5 @@ export async function resolveBenchmarkRequest({ github, context }) {
return null;
if (eventName === "pull_request" && pull.head.sha !== payload.pull_request.head.sha) return null;
if (!/^[a-f0-9]{40}$/u.test(pull.head.sha)) throw new Error("PR head must be an exact commit.");
- return request(harnesses, pull.head.sha, String(number));
+ return request(selection, pull.head.sha, String(number));
}
diff --git a/.github/scripts/resolve-benchmark-request.test.mjs b/.github/scripts/resolve-benchmark-request.test.mjs
index 823725f77..3fe444206 100644
--- a/.github/scripts/resolve-benchmark-request.test.mjs
+++ b/.github/scripts/resolve-benchmark-request.test.mjs
@@ -3,12 +3,16 @@ import { readFile } from "node:fs/promises";
import test from "node:test";
import {
+ BENCHMARK_DATASETS,
BENCHMARK_HARNESSES,
+ DEFAULT_DATASET,
DEFAULT_HARNESSES,
resolveBenchmarkRequest,
} from "./resolve-benchmark-request.mjs";
-const defaults = { harness: "eve-code,opencode,pi", slug: "eve-code+opencode+pi" };
+const sweLean = { dataset: "swe-lean", reasoning: "low", attempts: "5" };
+const deepsweLean = { dataset: "deepswe-lean", reasoning: "high", attempts: "3" };
+const defaults = { harness: "eve-code,opencode,pi", slug: "eve-code+opencode+pi", ...sweLean };
const sha = "a".repeat(40);
const pull = {
@@ -82,6 +86,7 @@ for (const harness of BENCHMARK_HARNESSES) {
assert.deepEqual(await resolve(fixture({ body: `/benchmark ${harness}` })), {
harness,
slug: harness,
+ ...sweLean,
sha,
pr: "42",
});
@@ -100,6 +105,45 @@ test("a comment can select several harnesses, comma- or space-separated, dedupli
}
});
+test("manual dispatch offers exactly the datasets the resolver accepts", async () => {
+ const workflow = await readFile(
+ new URL("../workflows/eve-code-benchmark.yml", import.meta.url),
+ "utf8",
+ );
+ const input = /\n {6}dataset:\n(?: {8}.*\n)+/u.exec(workflow)?.[0] ?? "";
+ const options = /^ {8}options: \[(.*)\]$/mu.exec(input)?.[1].split(/, */u);
+ assert.deepEqual(options, Object.keys(BENCHMARK_DATASETS));
+ assert.equal(/^ {8}default: (\S+)$/mu.exec(input)?.[1], DEFAULT_DATASET);
+});
+
+test("a comment can select a dataset, with the default harnesses or its own list", async () => {
+ assert.deepEqual(await resolve(fixture({ body: "/benchmark deepswe-lean" })), {
+ ...defaults,
+ ...deepsweLean,
+ sha,
+ pr: "42",
+ });
+ for (const body of [
+ "/benchmark deepswe-lean eve-code,pi",
+ "/benchmark eve-code pi deepswe-lean",
+ "/benchmark eve-code, deepswe-lean, pi",
+ ]) {
+ const result = await resolve(fixture({ body }));
+ assert.deepEqual(
+ [result.dataset, result.reasoning, result.attempts, result.harness, result.slug],
+ ["deepswe-lean", "high", "3", "eve-code,pi", "eve-code+pi"],
+ );
+ }
+ assert.equal((await resolve(fixture({ body: "/benchmark swe-lean pi" }))).dataset, "swe-lean");
+});
+
+test("a comment cannot name two datasets", async () => {
+ await assert.rejects(
+ resolve(fixture({ body: "/benchmark swe-lean deepswe-lean" })),
+ /at most one benchmark dataset/u,
+ );
+});
+
for (const permission of ["read", "triage", "none"]) {
test(`${permission} permission cannot launch a benchmark`, async () => {
const input = fixture({ permission });
@@ -204,6 +248,19 @@ test("manual dispatch defaults to the default harnesses and validates an explici
await assert.rejects(resolve(input), /Supported benchmark harnesses/u);
});
+test("manual dispatch can publish a deepswe-lean result and rejects unknown datasets", async () => {
+ const input = fixture({ eventName: "workflow_dispatch" });
+ input.context.payload.inputs = { harness: "", dataset: "deepswe-lean" };
+ assert.deepEqual(await resolve(input), {
+ ...defaults,
+ ...deepsweLean,
+ sha: input.context.sha,
+ pr: "",
+ });
+ input.context.payload.inputs.dataset = "terminal-bench";
+ await assert.rejects(resolve(input), /Supported benchmark datasets: swe-lean, deepswe-lean/u);
+});
+
test("pushes to main publish the default harnesses at the pushed commit", async () => {
const input = fixture({ eventName: "push" });
assert.deepEqual(await resolve(input), { ...defaults, sha: input.context.sha, pr: "" });
@@ -226,14 +283,18 @@ test("workflow names and concurrency preserve independent authorized harness run
workflow,
/needs: request\n\s+if: needs\.request\.outputs\.run == 'true'\n\s+concurrency:/u,
);
+ // A deepswe-lean run never cancels or overwrites a swe-lean run of the same harnesses.
+ assert.match(
+ workflow,
+ /group: eve-code-.*needs\.request\.outputs\.pr.*needs\.request\.outputs\.dataset.*needs\.request\.outputs\.slug/u,
+ );
assert.match(
workflow,
- /group: eve-code-.*needs\.request\.outputs\.pr.*needs\.request\.outputs\.slug/u,
+ /artifact-name: \$\{\{ needs\.request\.outputs\.dataset \}\}-\$\{\{ needs\.request\.outputs\.slug \}\}/u,
);
- assert.match(workflow, /artifact-name: swe-lean-\$\{\{ needs\.request\.outputs\.slug \}\}/u);
assert.match(workflow, /push:\n\s+branches: \[main\]/u);
- // Five attempts per task, run in waves of the action's maximum of 16 sandboxes.
- assert.match(workflow, /attempts: "5"/u);
+ // Each dataset's attempts run in waves of the action's maximum of 16 sandboxes.
+ assert.match(workflow, /attempts: \$\{\{ needs\.request\.outputs\.attempts \}\}/u);
assert.match(workflow, /concurrency: "16"/u);
assert.match(workflow, /agent-ref: \$\{\{ needs\.request\.outputs\.sha \}\}/u);
});
@@ -249,11 +310,11 @@ test("the consumer tracks eve-bench main and owns its model selection", async ()
workflow,
/model: \$\{\{ vars\.EVE_CODE_BENCH_MODEL \|\| 'anthropic\/claude-sonnet-5\.5' \}\}/u,
);
- assert.match(workflow, /reasoning: low\n/u);
+ assert.match(workflow, /reasoning: \$\{\{ needs\.request\.outputs\.reasoning \}\}\n/u);
assert.doesNotMatch(workflow, /runner-revision:/u);
assert.match(workflow, /blob-token: \$\{\{ secrets\.EVE_BENCH_BLOB_READ_WRITE_TOKEN \}\}/u);
assert.doesNotMatch(workflow, /artifact-id/u);
- assert.match(workflow, /dataset: swe-lean\n/u);
+ assert.match(workflow, /dataset: \$\{\{ needs\.request\.outputs\.dataset \}\}\n/u);
assert.match(workflow, /contenders: eve-code@baseline,eve-code@head\n/u);
assert.doesNotMatch(workflow, /^\s+task:/mu);
assert.equal((workflow.match(/uses: \.\/\.eve-bench-action/gu) ?? []).length, 1);
diff --git a/.github/workflows/eve-code-benchmark.yml b/.github/workflows/eve-code-benchmark.yml
index 1cae8b964..d844cc497 100644
--- a/.github/workflows/eve-code-benchmark.yml
+++ b/.github/workflows/eve-code-benchmark.yml
@@ -6,8 +6,10 @@ on:
- ".github/workflows/eve-code-benchmark.yml"
- ".github/scripts/resolve-benchmark-request*"
- "packages/eve-code/**"
- # Every eve-code change on main publishes a fresh result per harness, so PRs
- # compare their HEAD against the latest main as the baseline.
+ # Every eve-code change on main publishes a fresh swe-lean result per harness,
+ # so PRs compare their HEAD against the latest main as the baseline. Pushes do
+ # not run deepswe-lean: its baseline is only as fresh as the last manual
+ # dispatch, so a PR's delta there also includes eve-code changes merged since.
push:
branches: [main]
paths:
@@ -17,9 +19,14 @@ on:
workflow_dispatch:
inputs:
harness:
- description: Comma-separated harnesses to benchmark on SWE-lean
+ description: Comma-separated harnesses to benchmark
type: string
default: eve-code,opencode,pi
+ dataset:
+ description: eve-bench dataset to run
+ type: choice
+ options: [swe-lean, deepswe-lean]
+ default: swe-lean
# OIDC alternative: add `id-token: write` here and omit `vercel-token` below
# after configuring a matching Vercel team OIDC policy.
@@ -42,6 +49,9 @@ jobs:
run: ${{ steps.request.outputs.run }}
harness: ${{ steps.request.outputs.harness }}
slug: ${{ steps.request.outputs.slug }}
+ dataset: ${{ steps.request.outputs.dataset }}
+ reasoning: ${{ steps.request.outputs.reasoning }}
+ attempts: ${{ steps.request.outputs.attempts }}
sha: ${{ steps.request.outputs.sha }}
pr: ${{ steps.request.outputs.pr }}
steps:
@@ -64,12 +74,13 @@ jobs:
needs: request
if: needs.request.outputs.run == 'true'
concurrency:
- group: eve-code-${{ needs.request.outputs.pr || github.ref }}-${{ needs.request.outputs.slug }}
+ group: eve-code-${{ needs.request.outputs.pr || github.ref }}-${{ needs.request.outputs.dataset }}-${{ needs.request.outputs.slug }}
cancel-in-progress: true
runs-on: ubuntu-latest
- # Trials run in waves of up to 16 sandboxes; each trial can take up to the
- # 15-minute agent timeout plus preparation and verification.
- timeout-minutes: 90
+ # Trials run in waves of up to 16 sandboxes; each trial can take up to its
+ # dataset's agent timeout (20 minutes on deepswe-lean) plus preparation and
+ # verification.
+ timeout-minutes: 120
steps:
# vercel/eve is public and eve-bench is private; GitHub never lets a public
# repository `uses:` a private action, so fetch main with a read-only deploy
@@ -89,24 +100,24 @@ jobs:
agent-repository: vercel/eve
agent-paths: '["packages/eve-code"]'
agent-ref: ${{ needs.request.outputs.sha }}
- dataset: swe-lean
+ dataset: ${{ needs.request.outputs.dataset }}
# The PR comment's ranking tables show these plus each dimension's winner.
contenders: eve-code@baseline,eve-code@head
# Sonnet keeps Anthropic's prompt-cache behavior (tools first, cache writes)
# visible at half Opus's price. Avoid Gemini: opencode drops its tool-call
# thought signatures, so it stops reasoning after the first steps there.
model: ${{ vars.EVE_CODE_BENCH_MODEL || 'anthropic/claude-sonnet-5.5' }}
- reasoning: low
- # One attempt per task could not separate a 10-15% token change from
- # trial noise; five gives the paired bootstrap enough samples.
- attempts: "5"
- # The action's maximum. 7 tasks x 5 attempts per contender run in waves.
+ # Reasoning and attempts are calibrated per dataset; see
+ # .github/scripts/resolve-benchmark-request.mjs.
+ reasoning: ${{ needs.request.outputs.reasoning }}
+ attempts: ${{ needs.request.outputs.attempts }}
+ # The action's maximum; each contender's tasks x attempts run in waves.
concurrency: "16"
native-build: "false"
# Baselines and cross-harness references resolve automatically from the
# durable result store; only dispatch/push/schedule runs publish to it.
blob-token: ${{ secrets.EVE_BENCH_BLOB_READ_WRITE_TOKEN }}
- artifact-name: swe-lean-${{ needs.request.outputs.slug }}
+ artifact-name: ${{ needs.request.outputs.dataset }}-${{ needs.request.outputs.slug }}
gateway-key: ${{ secrets.AI_GATEWAY_API_KEY }}
github-token: ${{ github.token }}
vercel-token: ${{ secrets.VERCEL_TOKEN }}