Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

ci: publish nightly model benchmark results from main (#5189)

prhabfb293cf92a0
diff --git a/.github/scripts/discover-e2e-fixtures.mjs b/.github/scripts/discover-e2e-fixtures.mjs
index 31c98af3c..ff718cb8c 100644
--- a/.github/scripts/discover-e2e-fixtures.mjs
+++ b/.github/scripts/discover-e2e-fixtures.mjs
@@ -146,7 +146,7 @@ if (process.argv[1] === fileURLToPath(import.meta.url)) {
else process.stdout.write(result.lines);
}
-function discoverEvalIds(root) {
+export function discoverEvalIds(root) {
const evalIds = [];
const visit = (directory) => {
for (const entry of readdirSync(directory, { withFileTypes: true }).sort((a, b) =>
diff --git a/.github/scripts/stage-benchmark-leg.sh b/.github/scripts/stage-benchmark-leg.sh
new file mode 100755
index 000000000..6a23321f8
--- /dev/null
+++ b/.github/scripts/stage-benchmark-leg.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# Stages one eval-benchmark leg for upload as $RUNNER_TEMP/leg:
+# meta.json leg identity and runner context
+# expected.json `eve eval --list --json` output, when the run got that far
+# run/ the leg's single `.eve/evals/<timestamp>` tree, when one exists
+# The publish job (scripts/eval-benchmark/records.mjs) reads this layout.
+set -euo pipefail
+
+leg="$RUNNER_TEMP/leg"
+rm -rf "$leg"
+mkdir -p "$leg"
+
+jq -n \
+ --arg leg_id "$LEG_ID" \
+ --arg sha "$PLANNED_SHA" \
+ --arg model_id "$MODEL_ID" \
+ --arg world "$WORLD" \
+ --argjson attempt "$ATTEMPT" \
+ --arg runner_os "$RUNNER_OS_NAME" \
+ '{leg_id: $leg_id, sha: $sha, model_id: $model_id, world: $world, attempt: $attempt, runner_os: $runner_os}' \
+ >"$leg/meta.json"
+
+if [ -s "$RUNNER_TEMP/expected.json" ]; then
+ cp "$RUNNER_TEMP/expected.json" "$leg/expected.json"
+fi
+
+shopt -s nullglob
+runs=("$FIXTURE_DIR"/.eve/evals/*/)
+if [ "${#runs[@]}" -gt 1 ]; then
+ echo "::error::Expected one .eve/evals tree in $FIXTURE_DIR, found ${#runs[@]}."
+ exit 1
+fi
+if [ "${#runs[@]}" -eq 1 ]; then
+ cp -R "${runs[0]}" "$leg/run"
+fi
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 0f649d61c..1ab75cdc2 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -101,6 +101,9 @@ jobs:
- name: Test workflow and guard helpers
run: node --test .github/scripts/*.test.mjs scripts/hitl-state-readers.test.mjs
+ - name: Test eval benchmark scripts
+ run: pnpm test:eval-benchmark
+
test-integration:
name: test-integration (${{ matrix.os }})
runs-on: ${{ matrix.os }}
diff --git a/.github/workflows/eval-benchmark.yml b/.github/workflows/eval-benchmark.yml
new file mode 100644
index 000000000..60cbab1fd
--- /dev/null
+++ b/.github/workflows/eval-benchmark.yml
@@ -0,0 +1,361 @@
+name: Eval Benchmark
+
+# Tracks correctness, latency, and cost of fixture evals across models and eve
+# versions, and publishes records to private Blob. Runs only on trusted
+# `main` revisions and never on pull requests. Run legs hold model credentials
+# only; the publish job is the only one that holds the Blob token.
+on:
+ schedule:
+ # Nightly; skipped when main has not moved since the last successful
+ # scheduled run. Manual runs may be narrowed, so they never count.
+ - cron: "23 7 * * *"
+ workflow_dispatch:
+ inputs:
+ models:
+ description: Comma-separated e2e/benchmark.json model names (empty for all, "none" for mock legs only)
+ type: string
+ default: ""
+ fixtures:
+ description: Comma-separated e2e/benchmark.json fixture names (empty for all)
+ type: string
+ default: ""
+ attempts:
+ description: Attempts per fixture and model (empty for the registry default)
+ type: string
+ default: ""
+
+concurrency:
+ group: eval-benchmark-main
+ cancel-in-progress: false
+
+permissions:
+ contents: read
+
+jobs:
+ plan:
+ name: plan
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ actions: read
+ outputs:
+ run: ${{ steps.skip.outputs.run }}
+ sha: ${{ steps.skip.outputs.sha }}
+ live_matrix: ${{ steps.plan.outputs.live_matrix }}
+ mock_matrix: ${{ steps.plan.outputs.mock_matrix }}
+ max_parallel: ${{ steps.plan.outputs.max_parallel }}
+ evals_per_leg: ${{ steps.plan.outputs.evals_per_leg }}
+ steps:
+ - name: Require main
+ if: github.ref != 'refs/heads/main'
+ run: |
+ echo "::error::Benchmark results publish only from refs/heads/main (got ${GITHUB_REF})."
+ exit 1
+
+ - name: Checkout code
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+
+ - name: Setup Node.js
+ uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+ with:
+ node-version-file: .nvmrc
+
+ - name: Skip an unchanged nightly run
+ id: skip
+ env:
+ GH_TOKEN: ${{ github.token }}
+ EVENT_NAME: ${{ github.event_name }}
+ run: |
+ echo "sha=${GITHUB_SHA}" >> "$GITHUB_OUTPUT"
+ if [ "$EVENT_NAME" = "schedule" ]; then
+ last_sha="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow eval-benchmark.yml \
+ --branch main --event schedule --status success --limit 1 --json headSha --jq '.[0].headSha // ""')"
+ if [ "$last_sha" = "$GITHUB_SHA" ]; then
+ echo "main is still at the last benchmarked commit ${GITHUB_SHA}; skipping."
+ echo "run=false" >> "$GITHUB_OUTPUT"
+ exit 0
+ fi
+ fi
+ echo "run=true" >> "$GITHUB_OUTPUT"
+
+ - name: Plan legs
+ id: plan
+ if: steps.skip.outputs.run == 'true'
+ env:
+ BENCHMARK_MODELS: ${{ inputs.models }}
+ BENCHMARK_FIXTURES: ${{ inputs.fixtures }}
+ BENCHMARK_ATTEMPTS: ${{ inputs.attempts }}
+ run: node scripts/eval-benchmark/plan.mjs "$RUNNER_TEMP/plan.json" "$GITHUB_SHA"
+
+ - name: Upload plan
+ if: steps.skip.outputs.run == 'true'
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: benchmark-plan
+ path: ${{ runner.temp }}/plan.json
+ if-no-files-found: error
+ retention-days: 1
+
+ # Live-model legs: fixture × model × attempt on the local world. Eval
+ # failures are data, so neither the eval run nor the job fails the workflow.
+ # e2e/benchmark.json `concurrency` caps legs (gateway load) and evals per
+ # leg (runner contention, which skews latency).
+ run-live:
+ name: run-live (${{ matrix.fixture }}, ${{ matrix.model_id }}, ${{ matrix.attempt }})
+ needs: plan
+ if: needs.plan.outputs.run == 'true' && needs.plan.outputs.live_matrix != '[]'
+ runs-on: ubuntu-latest
+ continue-on-error: true
+ timeout-minutes: 90
+ strategy:
+ fail-fast: false
+ max-parallel: ${{ fromJSON(needs.plan.outputs.max_parallel) }}
+ matrix:
+ include: ${{ fromJSON(needs.plan.outputs.live_matrix) }}
+ env:
+ AI_GATEWAY_API_KEY: ${{ secrets.AI_GATEWAY_API_KEY }}
+ EVE_E2E_MODEL: ${{ matrix.model_id }}
+ EVALS_PER_LEG: ${{ needs.plan.outputs.evals_per_leg }}
+ EVE_SANDBOX_IMAGE_TAG: latest
+ steps:
+ - name: Checkout planned commit
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ ref: ${{ needs.plan.outputs.sha }}
+ persist-credentials: false
+
+ - name: Setup pnpm
+ uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
+
+ - name: Setup Node.js
+ uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+ with:
+ node-version-file: .nvmrc
+ cache: "pnpm"
+
+ - name: Install dependencies
+ run: pnpm install --frozen-lockfile
+
+ - name: Build eve package
+ run: pnpm --filter eve run build
+
+ - name: Build extension fixtures
+ # Mirrors e2e-local: extension fixtures ship no committed dist/.
+ run: |
+ pnpm --filter "./e2e/fixtures/*-extension" run --if-present build
+ pnpm --filter dist-extensions update gizmo-extension gadget-extension
+
+ - name: Run fixture evals
+ working-directory: ${{ matrix.dir }}
+ run: |
+ pnpm run --if-present e2e:prepare
+ pnpm exec eve eval --list --json > "$RUNNER_TEMP/expected.json"
+ if ! pnpm exec eve eval --verbose --max-concurrency "$EVALS_PER_LEG"; then
+ echo "::warning::Some evals failed; failures are recorded as benchmark data."
+ fi
+
+ - name: Stage leg artifact
+ if: always()
+ env:
+ LEG_ID: ${{ matrix.leg_id }}
+ FIXTURE_DIR: ${{ matrix.dir }}
+ PLANNED_SHA: ${{ needs.plan.outputs.sha }}
+ MODEL_ID: ${{ matrix.model_id }}
+ WORLD: ${{ matrix.world }}
+ ATTEMPT: ${{ matrix.attempt }}
+ RUNNER_OS_NAME: ${{ runner.os }}
+ run: bash .github/scripts/stage-benchmark-leg.sh
+
+ - name: Upload leg artifact
+ if: always()
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: benchmark-leg-${{ matrix.leg_id }}
+ path: ${{ runner.temp }}/leg
+ if-no-files-found: error
+ retention-days: 1
+
+ # Mock-model legs: one per fixture and workflow world. With no model latency
+ # or cost, their timings are close to pure eve overhead.
+ run-mock:
+ name: run-mock (${{ matrix.fixture }}, ${{ matrix.world }})
+ needs: plan
+ if: needs.plan.outputs.run == 'true' && needs.plan.outputs.mock_matrix != '[]'
+ runs-on: ubuntu-latest
+ continue-on-error: true
+ timeout-minutes: 60
+ strategy:
+ fail-fast: false
+ matrix:
+ include: ${{ fromJSON(needs.plan.outputs.mock_matrix) }}
+ # Unused by local-world legs; a service container cannot vary per leg.
+ services:
+ postgres:
+ image: postgres:18-alpine
+ env:
+ POSTGRES_DB: world
+ POSTGRES_PASSWORD: world
+ POSTGRES_USER: world
+ ports:
+ - 5432:5432
+ options: >-
+ --health-cmd "pg_isready --username=world --dbname=world"
+ --health-interval 5s
+ --health-timeout 5s
+ --health-retries 12
+ env:
+ # Mock models never reach the gateway.
+ AI_GATEWAY_API_KEY: invalid-e2e-world-suite-key
+ EVE_E2E_MODEL: mock
+ EVALS_PER_LEG: ${{ needs.plan.outputs.evals_per_leg }}
+ EVE_SANDBOX_IMAGE_TAG: latest
+ WORKFLOW_POSTGRES_MAX_POOL_SIZE: "52"
+ WORKFLOW_POSTGRES_URL: postgres://world:[email protected]:5432/world
+ WORKFLOW_POSTGRES_WORKER_CONCURRENCY: "50"
+ steps:
+ - name: Checkout planned commit
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ ref: ${{ needs.plan.outputs.sha }}
+ persist-credentials: false
+
+ - name: Setup pnpm
+ uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
+
+ - name: Setup Node.js
+ uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+ with:
+ node-version-file: .nvmrc
+ cache: "pnpm"
+
+ - name: Install dependencies
+ run: pnpm install --frozen-lockfile
+
+ - name: Build eve package
+ run: pnpm --filter eve run build
+
+ - name: Build extension fixtures
+ run: |
+ pnpm --filter "./e2e/fixtures/*-extension" run --if-present build
+ pnpm --filter dist-extensions update gizmo-extension gadget-extension
+
+ - name: Run fixture evals on the local world
+ if: matrix.world == 'local'
+ working-directory: ${{ matrix.dir }}
+ run: |
+ pnpm run --if-present e2e:prepare
+ pnpm exec eve eval --list --json --exclude-tag real-model > "$RUNNER_TEMP/expected.json"
+ if ! pnpm exec eve eval --verbose --exclude-tag real-model --max-concurrency "$EVALS_PER_LEG"; then
+ echo "::warning::Some evals failed; failures are recorded as benchmark data."
+ fi
+
+ - name: Run fixture evals on a production server
+ if: matrix.world != 'local'
+ working-directory: ${{ matrix.dir }}
+ run: |
+ set -euo pipefail
+ pnpm run --if-present e2e:prepare
+ pnpm exec eve eval --list --json --exclude-tag real-model > "$RUNNER_TEMP/expected.json"
+ if [ "$WORLD" = "postgres" ]; then pnpm exec bootstrap; fi
+ pnpm exec eve build
+ server_log="$RUNNER_TEMP/server.log"
+ EVE_DEV=1 pnpm exec eve start --host 127.0.0.1 --port 3000 >"$server_log" 2>&1 &
+ server_pid=$!
+ trap 'kill "$server_pid" 2>/dev/null; wait "$server_pid" 2>/dev/null || true' EXIT
+ ready=false
+ for _ in {1..120}; do
+ if curl --fail --silent http://127.0.0.1:3000/eve/v1/health >/dev/null; then
+ ready=true
+ break
+ fi
+ if ! kill -0 "$server_pid" 2>/dev/null; then cat "$server_log"; exit 1; fi
+ sleep 0.5
+ done
+ if [ "$ready" != "true" ]; then
+ cat "$server_log"
+ echo "::error::Timed out waiting for the eve server."
+ exit 1
+ fi
+ if ! pnpm exec eve eval --verbose --exclude-tag real-model --max-concurrency "$EVALS_PER_LEG" \
+ --url http://127.0.0.1:3000; then
+ echo "::warning::Some evals failed; failures are recorded as benchmark data."
+ fi
+ env:
+ WORLD: ${{ matrix.world }}
+ # Only non-local legs select a world; fixtures read an empty value
+ # as a package name, so local legs must leave it unset.
+ EVE_E2E_WORKFLOW_WORLD: ${{ matrix.world_package }}
+
+ - name: Stage leg artifact
+ if: always()
+ env:
+ LEG_ID: ${{ matrix.leg_id }}
+ FIXTURE_DIR: ${{ matrix.dir }}
+ PLANNED_SHA: ${{ needs.plan.outputs.sha }}
+ MODEL_ID: ${{ matrix.model_id }}
+ WORLD: ${{ matrix.world }}
+ ATTEMPT: ${{ matrix.attempt }}
+ RUNNER_OS_NAME: ${{ runner.os }}
+ run: bash .github/scripts/stage-benchmark-leg.sh
+
+ - name: Upload leg artifact
+ if: always()
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: benchmark-leg-${{ matrix.leg_id }}
+ path: ${{ runner.temp }}/leg
+ if-no-files-found: error
+ retention-days: 1
+
+ # Runs trusted code from the planned main commit, never code a leg uploaded.
+ # Planned legs without an artifact become `missing` records.
+ publish:
+ name: publish
+ needs: [plan, run-live, run-mock]
+ if: always() && needs.plan.result == 'success' && needs.plan.outputs.run == 'true'
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout planned commit
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ ref: ${{ needs.plan.outputs.sha }}
+ persist-credentials: false
+
+ - name: Setup pnpm
+ uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10
+
+ - name: Setup Node.js
+ uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+ with:
+ node-version-file: .nvmrc
+ cache: "pnpm"
+
+ - name: Install dependencies
+ run: pnpm install --frozen-lockfile
+
+ - name: Download plan
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ name: benchmark-plan
+ path: ${{ runner.temp }}/plan
+
+ - name: Prepare leg directory
+ run: mkdir -p "$RUNNER_TEMP/legs"
+
+ - name: Download leg artifacts
+ # No match is valid (every leg failed to upload); records mark those legs missing.
+ continue-on-error: true
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ pattern: benchmark-leg-*
+ path: ${{ runner.temp }}/legs
+
+ - name: Derive records
+ run: node scripts/eval-benchmark/records.mjs "$RUNNER_TEMP/plan/plan.json" "$RUNNER_TEMP/legs" "$RUNNER_TEMP/records.jsonl"
+
+ - name: Publish to Blob
+ env:
+ EVE_BENCHMARK_BLOB_READ_WRITE_TOKEN: ${{ secrets.EVE_BENCHMARK_BLOB_READ_WRITE_TOKEN }}
+ run: node scripts/eval-benchmark/publish.mjs "$RUNNER_TEMP/plan/plan.json" "$RUNNER_TEMP/legs" "$RUNNER_TEMP/records.jsonl"
diff --git a/e2e/benchmark.json b/e2e/benchmark.json
new file mode 100644
index 000000000..db4471134
--- /dev/null
+++ b/e2e/benchmark.json
@@ -0,0 +1,19 @@
+{
+ "$comment": "Registry for the model benchmark workflow (.github/workflows/eval-benchmark.yml), separate from e2e/matrix.json so adding a benchmark model never adds a CI check. `models` are slots: a stable `name` (`<lab>-high`, `-mid`, and `-low` tiers per major lab where the lab offers a current model for the tier, plus each open-weight family's main model; `openai-low` is eve's default model) and the gateway `id` that currently fills it. Bump a slot's `id` when its lab ships a successor; records keep both, and the pinned release marks the change. Leave out service-tier variants (`-fast`) unless they are eve's default. Each slot runs every listed fixture `attempts` times against the local world; each id must exist in the AI Gateway catalog, which pins its release. `mockWorlds` run each fixture once with mock models (`local`, or a world with a package in e2e/matrix.json). `concurrency.legs` caps live legs running at once (gateway load); `concurrency.evalsPerLeg` is passed to `eve eval --max-concurrency` on every leg, overriding a fixture's own maxConcurrency, so evals do not share a runner. Both change measured latency, and each run's manifest records them.",
+ "models": [
+ { "name": "openai-high", "id": "openai/gpt-6.1-sol" },
+ { "name": "openai-low", "id": "openai/gpt-6-luna-fast" },
+ { "name": "anthropic-high", "id": "anthropic/claude-opus-5.5" },
+ { "name": "anthropic-mid", "id": "anthropic/claude-sonnet-5.5" },
+ { "name": "anthropic-low", "id": "anthropic/claude-haiku-5.5" },
+ { "name": "google-mid", "id": "google/gemini-3.8-flash" },
+ { "name": "google-low", "id": "google/gemini-3.5-flash-lite" },
+ { "name": "open-kimi", "id": "moonshotai/kimi-k3" },
+ { "name": "open-glm", "id": "zai/glm-5.3" },
+ { "name": "open-minimax", "id": "minimax/minimax-m3" }
+ ],
+ "fixtures": ["agent-tools", "agent-subagents", "agent-tasks"],
+ "mockWorlds": ["local", "postgres"],
+ "attempts": 3,
+ "concurrency": { "legs": 8, "evalsPerLeg": 1 }
+}
diff --git a/package.json b/package.json
index 58ac7b840..ebbad9aa3 100644
--- a/package.json
+++ b/package.json
@@ -26,6 +26,7 @@
"test": "pnpm test:unit && pnpm test:integration",
"test:e2e": "turbo run test:e2e --concurrency=1",
"test:webchat": "pnpm --filter eve run test:webchat",
+ "test:eval-benchmark": "node --test scripts/eval-benchmark/*.test.mjs",
"test:framework-fixtures": "pnpm --filter eve run test:framework-fixtures",
"test:integration": "turbo run test:integration --filter=eve && turbo run test:integration --filter=\"!eve\"",
"test:scenario": "turbo run test:scenario",
@@ -39,6 +40,7 @@
"@changesets/cli": "3.0.1",
"@microsoft/api-extractor": "7.59.0",
"@types/node": "catalog:",
+ "@vercel/blob": "2.8.0",
"js-yaml": "4.3.2",
"oxfmt": "0.66.0",
"oxlint": "1.81.0",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 15b20794c..f2a63c2b6 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -293,6 +293,9 @@ importers:
'@types/node':
specifier: 'catalog:'
version: 24.13.3
+ '@vercel/blob':
+ specifier: 2.8.0
+ version: 2.8.0
js-yaml:
specifier: 4.3.2
version: 4.3.2
diff --git a/scripts/eval-benchmark/catalog.mjs b/scripts/eval-benchmark/catalog.mjs
new file mode 100644
index 000000000..ad6a10a2e
--- /dev/null
+++ b/scripts/eval-benchmark/catalog.mjs
@@ -0,0 +1,37 @@
+// Pins benchmark model ids to their AI Gateway catalog release, so a provider
+// re-pointing an alias shows up as an identity break instead of a trend.
+
+export const AI_GATEWAY_MODELS_URL = "https://ai-gateway.vercel.sh/v1/models";
+
+/**
+ * @param {{ name: string, id: string }[]} models
+ * @param {typeof fetch} [fetchImpl]
+ * @returns {Promise<{ name: string, id: string, release: string }[]>}
+ */
+export async function resolveModelReleases(models, fetchImpl = fetch) {
+ const response = await fetchImpl(AI_GATEWAY_MODELS_URL, { signal: AbortSignal.timeout(30_000) });
+ if (!response.ok) {
+ throw new Error(
+ `AI Gateway model catalog request failed (${response.status}); cannot pin model releases.`,
+ );
+ }
+ const body = await response.json();
+ if (!Array.isArray(body?.data)) throw new Error("AI Gateway returned an invalid model catalog.");
+ const catalog = new Map(body.data.map((entry) => [entry?.id, entry]));
+
+ return models.map(({ name, id }) => {
+ const entry = catalog.get(id);
+ if (entry === undefined) {
+ throw new Error(
+ `AI Gateway catalog has no model "${id}" (e2e/benchmark.json model "${name}"). Update the id or remove the entry.`,
+ );
+ }
+ // `released` is the catalog's release date in Unix seconds.
+ if (typeof entry.released !== "number" || !Number.isFinite(entry.released)) {
+ throw new Error(
+ `AI Gateway catalog entry "${id}" (e2e/benchmark.json model "${name}") has no release date to pin.`,
+ );
+ }
+ return { name, id, release: new Date(entry.released * 1000).toISOString().slice(0, 10) };
+ });
+}
diff --git a/scripts/eval-benchmark/digest.mjs b/scripts/eval-benchmark/digest.mjs
new file mode 100644
index 000000000..e85d9e784
--- /dev/null
+++ b/scripts/eval-benchmark/digest.mjs
@@ -0,0 +1,19 @@
+// `fixture_digest`: sha256 over the git tree ids of the fixture directory and
+// the shared e2e config at the planned commit. Hashing whole trees cannot miss
+// a helper an eval imports, at the cost of per-eval granularity: any edit to a
+// fixture breaks comparability for all of its evals. eve itself is excluded,
+// since eve's version is what benchmarks compare.
+import { execFileSync } from "node:child_process";
+import { createHash } from "node:crypto";
+
+const SHARED_PATHS = ["e2e/fixtures/e2e-config"];
+
+/** @returns {string} hex digest */
+export function fixtureDigest(sha, fixtureDir) {
+ const hash = createHash("sha256");
+ for (const path of [fixtureDir, ...SHARED_PATHS]) {
+ const tree = execFileSync("git", ["rev-parse", `${sha}:${path}`], { encoding: "utf8" }).trim();
+ hash.update(`${path}\0${tree}\n`);
+ }
+ return hash.digest("hex");
+}
diff --git a/scripts/eval-benchmark/plan.mjs b/scripts/eval-benchmark/plan.mjs
new file mode 100644
index 000000000..f54390b9a
--- /dev/null
+++ b/scripts/eval-benchmark/plan.mjs
@@ -0,0 +1,278 @@
+// Plans one benchmark run from e2e/benchmark.json:
+//
+// live_matrix attempt × fixture × model legs on the local world, ordered so
+// the legs running together spread across models
+// max_parallel, evals_per_leg
+// the registry's concurrency caps, which bound gateway load
+// and keep evals from sharing a runner
+// mock_matrix fixture × mock world legs, one attempt each
+// plan.json the resolved registry snapshot (models pinned to their AI
+// Gateway release, fixture digests, expected eval ids) plus
+// every leg,
+// handed to the publish job
+//
+// Dispatch inputs narrow the run through BENCHMARK_MODELS and
+// BENCHMARK_FIXTURES (comma-separated registry names) and BENCHMARK_ATTEMPTS.
+//
+// Usage: node scripts/eval-benchmark/plan.mjs <plan.json> [sha]
+import { execFileSync } from "node:child_process";
+import { appendFileSync, existsSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { discoverEvalIds } from "../../.github/scripts/discover-e2e-fixtures.mjs";
+import { resolveModelReleases } from "./catalog.mjs";
+import { fixtureDigest } from "./digest.mjs";
+
+export const PLAN_SCHEMA_VERSION = 1;
+const FIXTURE_ROOTS = ["e2e/fixtures", "apps/fixtures"];
+const NAME = /^[a-z0-9-]+$/;
+const MAX_ATTEMPTS = 10;
+const MAX_PARALLEL_LEGS = 50;
+const MAX_EVALS_PER_LEG = 16;
+
+/**
+ * Validate the registry and apply dispatch narrowing. Pure apart from the
+ * injected fixture lookup, so it is testable without a checkout.
+ *
+ * @param {{
+ * registry: any,
+ * matrix: any,
+ * findFixture: (name: string) => { dir: string, evalIds: string[], packageJson?: any } | undefined,
+ * narrow?: { models?: string, fixtures?: string, attempts?: string },
+ * }} input
+ */
+export function selectBenchmark({ registry, matrix, findFixture, narrow = {} }) {
+ const models = namedEntries(registry?.models, "models");
+ const attempts = positiveInteger(registry?.attempts, 'e2e/benchmark.json: "attempts"');
+ const concurrency = {
+ legs: boundedInteger(
+ registry?.concurrency?.legs,
+ MAX_PARALLEL_LEGS,
+ 'e2e/benchmark.json: "concurrency.legs"',
+ ),
+ evalsPerLeg: boundedInteger(
+ registry?.concurrency?.evalsPerLeg,
+ MAX_EVALS_PER_LEG,
+ 'e2e/benchmark.json: "concurrency.evalsPerLeg"',
+ ),
+ };
+ const fixtureNames = names(registry?.fixtures, "fixtures");
+ const mockWorlds = names(registry?.mockWorlds ?? [], "mockWorlds", { allowEmpty: true });
+ const matrixWorlds = new Map((matrix?.worlds ?? []).map((world) => [world.name, world]));
+ for (const world of mockWorlds) {
+ // Mock legs boot a local server, so a non-local world must be a package
+ // the fixture can select (EVE_E2E_WORKFLOW_WORLD), not a deploy target.
+ if (world !== "local" && matrixWorlds.get(world)?.package === undefined) {
+ const packaged = [...matrixWorlds.values()].filter((w) => w.package).map((w) => w.name);
+ throw new Error(
+ `e2e/benchmark.json: mock world "${world}" must be "local" or a world with a package in e2e/matrix.json (${packaged.join(", ")}).`,
+ );
+ }
+ }
+
+ const fixtures = fixtureNames.map((name) => {
+ const found = findFixture(name);
+ if (found === undefined) {
+ throw new Error(
+ `e2e/benchmark.json: fixture "${name}" was not found with an evals/ directory under ${FIXTURE_ROOTS.join(" or ")}.`,
+ );
+ }
+ if (found.evalIds.length === 0)
+ throw new Error(`e2e/benchmark.json: fixture "${name}" has no *.eval.ts files.`);
+ return { name, ...found };
+ });
+
+ const selectedModels = narrowNames(models, narrow.models, "model");
+ const selectedFixtures = narrowNames(fixtures, narrow.fixtures, "fixture");
+ const selectedAttempts =
+ narrow.attempts === undefined || narrow.attempts === ""
+ ? attempts
+ : positiveInteger(Number(narrow.attempts), "attempts input");
+
+ return {
+ attempts: selectedAttempts,
+ concurrency,
+ models: selectedModels,
+ fixtures: selectedFixtures,
+ mockWorlds: mockWorlds.map((name) => ({ name, package: matrixWorlds.get(name)?.package })),
+ };
+}
+
+/** Expand a selection into run legs. Leg ids are stable artifact-name suffixes. */
+export function planLegs({ models, fixtures, attempts, mockWorlds }) {
+ // GitHub starts matrix legs in order and cannot cap concurrency per model,
+ // so model-innermost order keeps each wave of legs spread across models and
+ // each eval's attempts spread over time.
+ const live = [];
+ for (let attempt = 1; attempt <= attempts; attempt += 1) {
+ for (const fixture of fixtures) {
+ for (const model of models) {
+ live.push(
+ leg({ kind: "live", fixture, model_name: model.name, model_id: model.id, attempt }),
+ );
+ }
+ }
+ }
+ const mock = [];
+ for (const fixture of fixtures) {
+ const selectedWorlds = fixture.packageJson?.e2e?.worlds;
+ for (const world of mockWorlds) {
+ // `local` is the default world every fixture boots on; other worlds
+ // follow the fixture's own e2e.worlds selection.
+ if (
+ world.name !== "local" &&
+ Array.isArray(selectedWorlds) &&
+ !selectedWorlds.includes(world.name)
+ )
+ continue;
+ mock.push(
+ leg({ kind: "mock", fixture, model_name: "mock", model_id: "mock", world, attempt: 1 }),
+ );
+ }
+ }
+ return { live, mock };
+}
+
+function leg({ kind, fixture, model_name, model_id, world = { name: "local" }, attempt }) {
+ return {
+ // Registry names never contain "_", so "__" keeps ids unambiguous.
+ leg_id: [kind, fixture.name, model_name, world.name, attempt].join("__"),
+ kind,
+ fixture: fixture.name,
+ dir: fixture.dir,
+ model_name,
+ model_id,
+ world: world.name,
+ world_package: world.package ?? "",
+ attempt,
+ };
+}
+
+function namedEntries(entries, key) {
+ if (!Array.isArray(entries) || entries.length === 0)
+ throw new Error(`e2e/benchmark.json: "${key}" must be a non-empty array.`);
+ const seen = new Set();
+ for (const entry of entries) {
+ if (typeof entry?.name !== "string" || !NAME.test(entry.name))
+ throw new Error(`e2e/benchmark.json: every "${key}" entry needs a lowercase-dashed "name".`);
+ if (typeof entry.id !== "string" || entry.id.length === 0)
+ throw new Error(`e2e/benchmark.json: "${key}" entry "${entry.name}" needs a gateway "id".`);
+ if (seen.has(entry.name) || seen.has(entry.id))
+ throw new Error(`e2e/benchmark.json: duplicate "${key}" entry "${entry.name}".`);
+ seen.add(entry.name).add(entry.id);
+ }
+ return entries.map(({ name, id }) => ({ name, id }));
+}
+
+function names(entries, key, { allowEmpty = false } = {}) {
+ if (!Array.isArray(entries) || (!allowEmpty && entries.length === 0))
+ throw new Error(`e2e/benchmark.json: "${key}" must be a non-empty array.`);
+ if (entries.some((entry) => typeof entry !== "string" || !NAME.test(entry)))
+ throw new Error(
+ `e2e/benchmark.json: "${key}" entries must be lowercase alphanumerics and dashes.`,
+ );
+ if (new Set(entries).size !== entries.length)
+ throw new Error(`e2e/benchmark.json: "${key}" contains duplicates.`);
+ return entries;
+}
+
+function positiveInteger(value, label) {
+ return boundedInteger(value, MAX_ATTEMPTS, label);
+}
+
+function boundedInteger(value, max, label) {
+ if (!Number.isInteger(value) || value < 1 || value > max)
+ throw new Error(`${label} must be an integer from 1 to ${max}.`);
+ return value;
+}
+
+/** An empty input keeps every entry; otherwise every requested name must exist. */
+function narrowNames(entries, input, label) {
+ if (input === undefined || input.trim() === "") return entries;
+ const requested = input
+ .split(",")
+ .map((name) => name.trim())
+ .filter(Boolean);
+ if (requested.length === 1 && requested[0] === "none") return [];
+ if (new Set(requested).size !== requested.length)
+ throw new Error(`Benchmark ${label} input "${input}" contains duplicates.`);
+ const known = new Map(entries.map((entry) => [entry.name, entry]));
+ const unknown = requested.filter((name) => !known.has(name));
+ if (unknown.length > 0) {
+ throw new Error(
+ `Unknown benchmark ${label}${unknown.length === 1 ? "" : "s"} ${unknown.join(", ")}; choose from ${[...known.keys()].join(", ")} (or "none").`,
+ );
+ }
+ return requested.map((name) => known.get(name));
+}
+
+function findFixtureOnDisk(name) {
+ for (const root of FIXTURE_ROOTS) {
+ const dir = join(root, name);
+ if (!existsSync(join(dir, "evals"))) continue;
+ const packageJsonPath = join(dir, "package.json");
+ return {
+ dir,
+ evalIds: discoverEvalIds(join(dir, "evals")),
+ packageJson: existsSync(packageJsonPath)
+ ? JSON.parse(readFileSync(packageJsonPath, "utf8"))
+ : undefined,
+ };
+ }
+ return undefined;
+}
+
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
+ const [planPath, shaArgument] = process.argv.slice(2);
+ if (planPath === undefined) {
+ console.error("Usage: node scripts/eval-benchmark/plan.mjs <plan.json> [sha]");
+ process.exit(1);
+ }
+ const sha =
+ shaArgument ?? execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }).trim();
+ const selection = selectBenchmark({
+ registry: JSON.parse(readFileSync("e2e/benchmark.json", "utf8")),
+ matrix: JSON.parse(readFileSync("e2e/matrix.json", "utf8")),
+ findFixture: findFixtureOnDisk,
+ narrow: {
+ models: process.env.BENCHMARK_MODELS,
+ fixtures: process.env.BENCHMARK_FIXTURES,
+ attempts: process.env.BENCHMARK_ATTEMPTS,
+ },
+ });
+ const models = await resolveModelReleases(selection.models);
+ const { live, mock } = planLegs({ ...selection, models });
+ const plan = {
+ schema_version: PLAN_SCHEMA_VERSION,
+ sha,
+ planned_at: new Date().toISOString(),
+ registry: {
+ attempts: selection.attempts,
+ // Concurrency changes measured latency, so every manifest records it.
+ concurrency: {
+ legs: selection.concurrency.legs,
+ evals_per_leg: selection.concurrency.evalsPerLeg,
+ },
+ models,
+ mock_worlds: selection.mockWorlds.map((world) => world.name),
+ fixtures: selection.fixtures.map(({ name, dir, evalIds }) => ({
+ name,
+ dir,
+ digest: fixtureDigest(sha, dir),
+ eval_ids: evalIds,
+ })),
+ },
+ legs: [...live, ...mock],
+ };
+ writeFileSync(planPath, `${JSON.stringify(plan, null, 2)}\n`);
+ const lines = [
+ `live_matrix=${JSON.stringify(live)}`,
+ `mock_matrix=${JSON.stringify(mock)}`,
+ `max_parallel=${selection.concurrency.legs}`,
+ `evals_per_leg=${selection.concurrency.evalsPerLeg}`,
+ ];
+ console.error(`Planned ${live.length} live and ${mock.length} mock legs at ${sha}.`);
+ if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, `${lines.join("\n")}\n`);
+ else process.stdout.write(`${lines.join("\n")}\n`);
+}
diff --git a/scripts/eval-benchmark/plan.test.mjs b/scripts/eval-benchmark/plan.test.mjs
new file mode 100644
index 000000000..48ef391f6
--- /dev/null
+++ b/scripts/eval-benchmark/plan.test.mjs
@@ -0,0 +1,151 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import { resolveModelReleases } from "./catalog.mjs";
+import { planLegs, selectBenchmark } from "./plan.mjs";
+
+const registry = {
+ models: [
+ { name: "alpha", id: "acme/alpha" },
+ { name: "beta", id: "acme/beta" },
+ ],
+ fixtures: ["agent-one", "agent-two"],
+ mockWorlds: ["local", "postgres"],
+ attempts: 2,
+ concurrency: { legs: 3, evalsPerLeg: 1 },
+};
+const matrix = {
+ worlds: [{ name: "vercel" }, { name: "postgres", package: "@workflow/world-postgres" }],
+};
+const fixtures = {
+ "agent-one": { dir: "e2e/fixtures/agent-one", evalIds: ["a", "b"] },
+ "agent-two": {
+ dir: "e2e/fixtures/agent-two",
+ evalIds: ["c"],
+ packageJson: { e2e: { worlds: ["vercel"] } },
+ },
+};
+const select = (overrides = {}, narrow = {}) =>
+ selectBenchmark({
+ registry: { ...registry, ...overrides },
+ matrix,
+ findFixture: (name) => fixtures[name],
+ narrow,
+ });
+
+test("expands fixtures × models × attempts and mock worlds the fixture selects", () => {
+ const selection = select();
+ assert.deepEqual(selection.concurrency, { legs: 3, evalsPerLeg: 1 });
+ const { live, mock } = planLegs(selection);
+ assert.equal(live.length, 2 * 2 * 2);
+ // Models vary fastest so legs started together spread across models, and
+ // each eval's attempts are spread over the run.
+ assert.deepEqual(
+ live.slice(0, 5).map((leg) => leg.leg_id),
+ [
+ "live__agent-one__alpha__local__1",
+ "live__agent-one__beta__local__1",
+ "live__agent-two__alpha__local__1",
+ "live__agent-two__beta__local__1",
+ "live__agent-one__alpha__local__2",
+ ],
+ );
+ assert.deepEqual(live[0], {
+ leg_id: "live__agent-one__alpha__local__1",
+ kind: "live",
+ fixture: "agent-one",
+ dir: "e2e/fixtures/agent-one",
+ model_name: "alpha",
+ model_id: "acme/alpha",
+ world: "local",
+ world_package: "",
+ attempt: 1,
+ });
+ assert.equal(new Set(live.map((leg) => leg.leg_id)).size, live.length);
+ assert.deepEqual(
+ mock.map((leg) => [leg.leg_id, leg.world_package]),
+ [
+ ["mock__agent-one__mock__local__1", ""],
+ ["mock__agent-one__mock__postgres__1", "@workflow/world-postgres"],
+ ["mock__agent-two__mock__local__1", ""],
+ ],
+ );
+});
+
+test("dispatch inputs narrow every dimension and reject unknown names", () => {
+ const selection = select({}, { models: "beta", fixtures: "agent-two", attempts: "1" });
+ assert.deepEqual(
+ planLegs(selection).live.map((leg) => leg.leg_id),
+ ["live__agent-two__beta__local__1"],
+ );
+ assert.deepEqual(planLegs(select({}, { fixtures: "none" })), { live: [], mock: [] });
+ assert.throws(
+ () => select({}, { models: "gamma" }),
+ /Unknown benchmark model gamma; choose from alpha, beta/,
+ );
+ assert.throws(
+ () => select({}, { models: "beta,beta" }),
+ /Benchmark model input "beta,beta" contains duplicates/,
+ );
+ assert.throws(
+ () => select({}, { attempts: "0" }),
+ /attempts input must be an integer from 1 to 10/,
+ );
+});
+
+test("registry errors name the offending entry", () => {
+ for (const [overrides, message] of [
+ [{ fixtures: ["agent-missing"] }, /fixture "agent-missing" was not found/],
+ [{ mockWorlds: ["moon"] }, /mock world "moon" must be "local" or a world with a package/],
+ [{ mockWorlds: ["vercel"] }, /mock world "vercel" must be "local" or a world with a package/],
+ [
+ {
+ models: [
+ { name: "alpha", id: "x" },
+ { name: "alpha", id: "y" },
+ ],
+ },
+ /duplicate "models" entry "alpha"/,
+ ],
+ [{ concurrency: undefined }, /"concurrency.legs" must be an integer from 1 to 50/],
+ [
+ { concurrency: { legs: 5, evalsPerLeg: 0 } },
+ /"concurrency.evalsPerLeg" must be an integer from 1 to 16/,
+ ],
+ ])
+ assert.throws(() => select(overrides), message);
+});
+
+test("the committed registry is valid", () => {
+ const committed = JSON.parse(readFileSync(new URL("../../e2e/benchmark.json", import.meta.url)));
+ const committedMatrix = JSON.parse(
+ readFileSync(new URL("../../e2e/matrix.json", import.meta.url)),
+ );
+ const selection = selectBenchmark({
+ registry: committed,
+ matrix: committedMatrix,
+ findFixture: () => ({ dir: "fixture", evalIds: ["eval"] }),
+ });
+ assert.equal(selection.models.length, committed.models.length);
+});
+
+const catalog =
+ (data, ok = true) =>
+ async () => ({ ok, status: ok ? 200 : 503, json: async () => ({ data }) });
+
+test("pins each model to its catalog release and fails before running on unknown ids", async () => {
+ const released = Date.UTC(2026, 8, 22) / 1000;
+ const entries = [{ id: "acme/alpha", released }];
+ assert.deepEqual(
+ await resolveModelReleases([{ name: "alpha", id: "acme/alpha" }], catalog(entries)),
+ [{ name: "alpha", id: "acme/alpha", release: "2026-09-22" }],
+ );
+ await assert.rejects(
+ resolveModelReleases([{ name: "beta", id: "acme/beta" }], catalog(entries)),
+ /AI Gateway catalog has no model "acme\/beta" \(e2e\/benchmark.json model "beta"\)/,
+ );
+ await assert.rejects(
+ resolveModelReleases([{ name: "alpha", id: "acme/alpha" }], catalog([{ id: "acme/alpha" }])),
+ /has no release date to pin/,
+ );
+});
diff --git a/scripts/eval-benchmark/publish.mjs b/scripts/eval-benchmark/publish.mjs
new file mode 100644
index 000000000..6e6845b75
--- /dev/null
+++ b/scripts/eval-benchmark/publish.mjs
@@ -0,0 +1,171 @@
+// Publishes one benchmark run to private Vercel Blob. Every object is
+// immutable; re-running with identical content is a no-op and different
+// content fails. manifest.json is written last, so its presence tells ingest
+// the run is complete:
+//
+// runs/<github_run_id>/<run_attempt>/raw/<fixture>/<model>/<world>/<attempt>/...
+// runs/<github_run_id>/<run_attempt>/records.jsonl
+// runs/<github_run_id>/<run_attempt>/manifest.json
+//
+// Usage: node scripts/eval-benchmark/publish.mjs <plan.json> <legs-dir> <records.jsonl>
+// Requires EVE_BENCHMARK_BLOB_READ_WRITE_TOKEN, GITHUB_RUN_ID, GITHUB_RUN_ATTEMPT.
+import { createHash } from "node:crypto";
+import { readdirSync, readFileSync } from "node:fs";
+import { join, relative } from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { RECORD_SCHEMA_VERSION, legArtifactPath, readLegs } from "./records.mjs";
+
+const UPLOAD_CONCURRENCY = 8;
+
+/**
+ * @typedef {{
+ * put: (pathname: string, body: Buffer, contentType: string) => Promise<void>,
+ * read: (pathname: string) => Promise<Buffer | null>,
+ * }} ImmutableStore `put` must reject with an "already exists" error rather than overwrite.
+ */
+
+/**
+ * @param {{ plan: any, legs: ReturnType<typeof readLegs>, recordsJsonl: string, githubRunId: string, runAttempt: number, eveVersion: string, store: ImmutableStore }} input
+ */
+export async function publishRun({
+ plan,
+ legs,
+ recordsJsonl,
+ githubRunId,
+ runAttempt,
+ eveVersion,
+ store,
+}) {
+ const records = recordsJsonl
+ .split("\n")
+ .filter(Boolean)
+ .map((line) => JSON.parse(line));
+ for (const record of records) {
+ if (record.github_run_id !== String(githubRunId) || record.run_attempt !== runAttempt)
+ throw new Error(`Record ${record.fixture}/${record.eval_id} belongs to another run.`);
+ }
+
+ const prefix = `runs/${githubRunId}/${runAttempt}`;
+ const uploads = legs.flatMap(({ leg, dir }) =>
+ dir === undefined
+ ? []
+ : listFiles(dir).map((path) => ({
+ pathname: `${prefix}/${legArtifactPath(leg)}/${relative(dir, path).replaceAll("\\", "/")}`,
+ path,
+ })),
+ );
+ await mapLimit(uploads, UPLOAD_CONCURRENCY, ({ pathname, path }) =>
+ putImmutable(store, pathname, readFileSync(path), contentTypeOf(path)),
+ );
+
+ const recordsBody = Buffer.from(recordsJsonl);
+ await putImmutable(store, `${prefix}/records.jsonl`, recordsBody, "application/x-ndjson");
+ const manifest = {
+ schema_version: RECORD_SCHEMA_VERSION,
+ github_run_id: String(githubRunId),
+ run_attempt: runAttempt,
+ sha: plan.sha,
+ eve_version: eveVersion,
+ planned_at: plan.planned_at,
+ registry: plan.registry,
+ legs: legs.map(({ leg, dir }) => ({ ...leg, uploaded: dir !== undefined })),
+ record_count: records.length,
+ records_sha256: createHash("sha256").update(recordsBody).digest("hex"),
+ };
+ await putImmutable(
+ store,
+ `${prefix}/manifest.json`,
+ Buffer.from(`${JSON.stringify(manifest, null, 2)}\n`),
+ "application/json",
+ );
+ return { prefix, objects: uploads.length + 2, records: records.length };
+}
+
+async function putImmutable(store, pathname, body, contentType) {
+ try {
+ await store.put(pathname, body, contentType);
+ } catch (error) {
+ if (!(error instanceof Error) || !error.message.includes("already exists")) throw error;
+ const published = await store.read(pathname);
+ if (published === null) throw new Error(`${pathname} exists but could not be read back.`);
+ if (!published.equals(body)) {
+ throw new Error(
+ `${pathname} was already published with different content; benchmark runs are immutable. Re-run the workflow to get a new run attempt instead.`,
+ );
+ }
+ }
+}
+
+function listFiles(dir) {
+ return readdirSync(dir, { withFileTypes: true, recursive: true })
+ .filter((entry) => entry.isFile())
+ .map((entry) => join(entry.parentPath, entry.name))
+ .sort();
+}
+
+function contentTypeOf(path) {
+ if (path.endsWith(".json")) return "application/json";
+ if (path.endsWith(".ndjson") || path.endsWith(".jsonl")) return "application/x-ndjson";
+ return "application/octet-stream";
+}
+
+async function mapLimit(items, limit, task) {
+ let next = 0;
+ const workers = Array.from({ length: Math.min(limit, items.length) }, async () => {
+ while (next < items.length) await task(items[next++]);
+ });
+ await Promise.all(workers);
+}
+
+/** Private, immutable Blob store backed by @vercel/blob. */
+export async function vercelBlobStore(token) {
+ const { get, put } = await import("@vercel/blob");
+ return {
+ async put(pathname, body, contentType) {
+ await put(pathname, body, {
+ token,
+ access: "private",
+ addRandomSuffix: false,
+ allowOverwrite: false,
+ cacheControlMaxAge: 31_536_000,
+ contentType,
+ });
+ },
+ async read(pathname) {
+ const result = await get(pathname, { token, access: "private", useCache: false });
+ return result === null ? null : Buffer.from(await new Response(result.stream).arrayBuffer());
+ },
+ };
+}
+
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
+ const [planPath, legsDir, recordsPath] = process.argv.slice(2);
+ if (recordsPath === undefined) {
+ console.error(
+ "Usage: node scripts/eval-benchmark/publish.mjs <plan.json> <legs-dir> <records.jsonl>",
+ );
+ process.exit(1);
+ }
+ const token = process.env.EVE_BENCHMARK_BLOB_READ_WRITE_TOKEN;
+ if (!token)
+ throw new Error(
+ "EVE_BENCHMARK_BLOB_READ_WRITE_TOKEN is required to publish benchmark results.",
+ );
+ const githubRunId = process.env.GITHUB_RUN_ID;
+ if (githubRunId === undefined)
+ throw new Error("GITHUB_RUN_ID is required to key the published run.");
+ const plan = JSON.parse(readFileSync(planPath, "utf8"));
+ const result = await publishRun({
+ plan,
+ legs: readLegs(plan, legsDir),
+ recordsJsonl: readFileSync(recordsPath, "utf8"),
+ githubRunId,
+ runAttempt: Number(process.env.GITHUB_RUN_ATTEMPT ?? "1"),
+ eveVersion: JSON.parse(readFileSync("packages/eve/package.json", "utf8")).version,
+ store: await vercelBlobStore(token),
+ });
+ console.error(
+ `Published ${result.records} records and ${result.objects} objects to ${result.prefix}.`,
+ );
+}
diff --git a/scripts/eval-benchmark/records.mjs b/scripts/eval-benchmark/records.mjs
new file mode 100644
index 000000000..12a27d5c6
--- /dev/null
+++ b/scripts/eval-benchmark/records.mjs
@@ -0,0 +1,191 @@
+// Derives records.jsonl from the run legs' uploaded artifacts. Each leg
+// artifact (downloaded to <legs-dir>/benchmark-leg-<leg_id>/) holds:
+//
+// meta.json { leg_id, sha, runner_os } written by the run job
+// expected.json `eve eval --list --json` under the leg's tag filters
+// run/ the leg's single `.eve/evals/<timestamp>` tree
+//
+// Each record is one attempt of one eval on one model and world, keyed by
+// (github_run_id, run_attempt, fixture, eval_id, model_id, world, attempt).
+// Results are comparable across runs when (eval_id, fixture_digest, model_id,
+// model_release, world) match. Planned legs that uploaded nothing, and expected
+// evals with no result, become `missing` records so coverage stays honest.
+// Richer measurements can be derived later from the published raw trees.
+//
+// A live leg's eval whose model steps all ran on another model (fixtures script
+// some delegation evals with eve's mock models) says nothing about the leg's
+// model, so it is excluded; the mock track still measures it.
+//
+// Usage: node scripts/eval-benchmark/records.mjs <plan.json> <legs-dir> <records.jsonl>
+import { existsSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+export const RECORD_SCHEMA_VERSION = 1;
+export const LEG_ARTIFACT_PREFIX = "benchmark-leg-";
+
+/** Map downloaded leg directories onto planned legs, rejecting anything unplanned. */
+export function readLegs(plan, legsDir) {
+ const planned = new Map(plan.legs.map((leg) => [leg.leg_id, leg]));
+ const found = new Map();
+ const entries = existsSync(legsDir) ? readdirSync(legsDir, { withFileTypes: true }) : [];
+ for (const entry of entries) {
+ if (!entry.isDirectory() || !entry.name.startsWith(LEG_ARTIFACT_PREFIX))
+ throw new Error(
+ `Unexpected entry "${entry.name}" in ${legsDir}; expected ${LEG_ARTIFACT_PREFIX}<leg_id> directories.`,
+ );
+ const dir = join(legsDir, entry.name);
+ const metaPath = join(dir, "meta.json");
+ if (!existsSync(metaPath)) throw new Error(`${dir} has no meta.json.`);
+ const meta = JSON.parse(readFileSync(metaPath, "utf8"));
+ if (`${LEG_ARTIFACT_PREFIX}${meta.leg_id}` !== entry.name)
+ throw new Error(`${metaPath}: leg_id "${meta.leg_id}" does not match its artifact name.`);
+ if (!planned.has(meta.leg_id))
+ throw new Error(`${metaPath}: leg "${meta.leg_id}" was not planned.`);
+ if (meta.sha !== plan.sha)
+ throw new Error(`${metaPath}: ran at ${meta.sha}, but the plan is for ${plan.sha}.`);
+ const expectedPath = join(dir, "expected.json");
+ found.set(meta.leg_id, {
+ dir,
+ meta,
+ expected: existsSync(expectedPath)
+ ? JSON.parse(readFileSync(expectedPath, "utf8")).map((entry) => entry.id)
+ : undefined,
+ });
+ }
+ return plan.legs.map((leg) => ({ leg, ...found.get(leg.leg_id) }));
+}
+
+/** Path of a leg's raw tree, relative to the run's Blob prefix. */
+export function legArtifactPath(leg) {
+ return `raw/${leg.fixture}/${leg.model_name}/${leg.world}/${leg.attempt}`;
+}
+
+/**
+ * @param {{
+ * plan: any,
+ * legs: ReturnType<typeof readLegs>,
+ * githubRunId: string,
+ * runAttempt: number,
+ * }} input
+ */
+export function buildRecords({ plan, legs, githubRunId, runAttempt }) {
+ const excluded = [];
+ const fixtures = new Map(plan.registry.fixtures.map((fixture) => [fixture.name, fixture]));
+ const releases = new Map(plan.registry.models.map((model) => [model.id, model.release]));
+ const attempts = [];
+ for (const { leg, dir, meta, expected } of legs) {
+ const fixture = fixtures.get(leg.fixture);
+ const artifacts = dir === undefined ? [] : listEvalArtifacts(join(dir, "run"));
+ const results = new Map(
+ artifacts.map((path) => {
+ const artifact = JSON.parse(readFileSync(path, "utf8"));
+ return [artifact.id, artifact];
+ }),
+ );
+ const missingReason = dir === undefined ? "job-missing" : "eval-missing";
+ const expectedIds = expected ?? fixture.eval_ids;
+ for (const evalId of new Set([...expectedIds, ...results.keys()])) {
+ const artifact = results.get(evalId);
+ if (leg.kind === "live" && artifact !== undefined && !ranOnModel(artifact, leg.model_id)) {
+ excluded.push({ leg_id: leg.leg_id, eval_id: evalId });
+ continue;
+ }
+ attempts.push({
+ leg,
+ meta,
+ evalId,
+ artifact,
+ missingReason,
+ uploaded: dir !== undefined,
+ });
+ }
+ }
+
+ const records = attempts.map(({ leg, meta, evalId, artifact, missingReason, uploaded }) => {
+ const startedAt = artifact?.startedAt ?? null;
+ const completedAt = artifact?.completedAt ?? null;
+ return {
+ schema_version: RECORD_SCHEMA_VERSION,
+ github_run_id: String(githubRunId),
+ run_attempt: runAttempt,
+ fixture: leg.fixture,
+ eval_id: evalId,
+ model_id: leg.model_id,
+ model_slot: leg.model_name,
+ model_release: leg.kind === "mock" ? null : (releases.get(leg.model_id) ?? null),
+ world: leg.world,
+ attempt: leg.attempt,
+ fixture_digest: fixtures.get(leg.fixture).digest,
+ sha: plan.sha,
+ eve_version: artifact?.result?.runtimeIdentity?.eveVersion ?? null,
+ runner_os: meta?.runner_os ?? null,
+ missing: artifact === undefined ? missingReason : null,
+ verdict: artifact?.verdict ?? null,
+ assertions: (artifact?.assertions ?? []).map(({ name, severity, score, passed }) => ({
+ name,
+ severity,
+ score,
+ passed,
+ })),
+ error: artifact?.error ?? null,
+ started_at: startedAt,
+ completed_at: completedAt,
+ duration_ms:
+ startedAt !== null && completedAt !== null
+ ? Date.parse(completedAt) - Date.parse(startedAt)
+ : null,
+ artifact_path: uploaded ? legArtifactPath(leg) : null,
+ };
+ });
+ return { records, excluded };
+}
+
+/**
+ * False only when the eval made model calls and none used `modelId`. An eval
+ * that never reached a model step stays, so its failure is recorded.
+ */
+function ranOnModel(artifact, modelId) {
+ const models = artifact.result?.derived?.models ?? [];
+ return models.length === 0 || models.includes(modelId);
+}
+
+/** Find every eval detail artifact under one `.eve/evals/<timestamp>` tree. */
+export function listEvalArtifacts(runDirectory) {
+ const evalsDirectory = join(runDirectory, "evals");
+ if (!existsSync(evalsDirectory)) return [];
+ return readdirSync(evalsDirectory, { withFileTypes: true, recursive: true })
+ .filter((entry) => entry.isFile() && entry.name.endsWith(".json"))
+ .map((entry) => join(entry.parentPath, entry.name))
+ .sort();
+}
+
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
+ const [planPath, legsDir, outPath] = process.argv.slice(2);
+ if (outPath === undefined) {
+ console.error(
+ "Usage: node scripts/eval-benchmark/records.mjs <plan.json> <legs-dir> <records.jsonl>",
+ );
+ process.exit(1);
+ }
+ const githubRunId = process.env.GITHUB_RUN_ID;
+ const runAttempt = Number(process.env.GITHUB_RUN_ATTEMPT ?? "1");
+ if (githubRunId === undefined)
+ throw new Error("GITHUB_RUN_ID is required to key benchmark records.");
+ const plan = JSON.parse(readFileSync(planPath, "utf8"));
+ const { records, excluded } = buildRecords({
+ plan,
+ legs: readLegs(plan, legsDir),
+ githubRunId,
+ runAttempt,
+ });
+ writeFileSync(outPath, records.map((record) => `${JSON.stringify(record)}\n`).join(""));
+ const missing = records.filter((record) => record.missing !== null).length;
+ console.error(`Wrote ${records.length} records (${missing} missing) to ${outPath}.`);
+ if (excluded.length > 0) {
+ console.error(
+ `Excluded ${excluded.length} live-leg evals whose model steps never used the leg's model:`,
+ );
+ for (const { leg_id, eval_id } of excluded) console.error(` ${leg_id} ${eval_id}`);
+ }
+}
diff --git a/scripts/eval-benchmark/records.test.mjs b/scripts/eval-benchmark/records.test.mjs
new file mode 100644
index 000000000..31c725c62
--- /dev/null
+++ b/scripts/eval-benchmark/records.test.mjs
@@ -0,0 +1,205 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { publishRun } from "./publish.mjs";
+import { buildRecords, readLegs } from "./records.mjs";
+
+const SHA = "a".repeat(40);
+const plan = {
+ schema_version: 1,
+ sha: SHA,
+ planned_at: "2026-01-01T00:00:00.000Z",
+ registry: {
+ attempts: 1,
+ models: [{ name: "alpha", id: "acme/alpha", release: "2026-09-22" }],
+ mock_worlds: ["local"],
+ fixtures: [
+ {
+ name: "agent-tools",
+ dir: "e2e/fixtures/agent-tools",
+ digest: "f".repeat(64),
+ eval_ids: ["static-tools/ends-turn-function", "static-tools/no-reply"],
+ },
+ ],
+ },
+ legs: [
+ {
+ leg_id: "live__agent-tools__alpha__local__1",
+ kind: "live",
+ fixture: "agent-tools",
+ dir: "e2e/fixtures/agent-tools",
+ model_name: "alpha",
+ model_id: "acme/alpha",
+ world: "local",
+ world_package: "",
+ attempt: 1,
+ },
+ {
+ leg_id: "mock__agent-tools__mock__local__1",
+ kind: "mock",
+ fixture: "agent-tools",
+ dir: "e2e/fixtures/agent-tools",
+ model_name: "mock",
+ model_id: "mock",
+ world: "local",
+ world_package: "",
+ attempt: 1,
+ },
+ ],
+};
+
+/** One uploaded live leg whose run produced one of its two expected evals; the mock leg is missing. */
+function legsDirectory(t, meta = {}) {
+ const root = mkdtempSync(join(tmpdir(), "eve-benchmark-"));
+ t.after(() => rmSync(root, { recursive: true, force: true }));
+ const leg = join(root, "benchmark-leg-live__agent-tools__alpha__local__1");
+ mkdirSync(join(leg, "run", "evals", "static-tools"), { recursive: true });
+ writeFileSync(
+ join(leg, "meta.json"),
+ JSON.stringify({
+ leg_id: "live__agent-tools__alpha__local__1",
+ sha: SHA,
+ runner_os: "Linux",
+ ...meta,
+ }),
+ );
+ writeEval(leg, "static-tools/ends-turn-function", "acme/alpha");
+ writeFileSync(join(leg, "run", "summary.json"), "{}");
+ return root;
+}
+
+function writeEval(leg, evalId, modelId) {
+ const artifact = {
+ id: evalId,
+ verdict: "passed",
+ assertions: [{ name: "replied", severity: "gate", score: 1, passed: true, extra: "x" }],
+ startedAt: "2026-01-01T00:00:00.000Z",
+ completedAt: "2026-01-01T00:00:01.500Z",
+ result: {
+ runtimeIdentity: { eveVersion: "0.71.2" },
+ derived: { models: [modelId] },
+ },
+ };
+ writeFileSync(join(leg, "run", "evals", `${evalId}.json`), JSON.stringify(artifact));
+}
+
+const build = (legsDir) => buildWithExclusions(legsDir).records;
+const buildWithExclusions = (legsDir) =>
+ buildRecords({
+ plan,
+ legs: readLegs(plan, legsDir),
+ githubRunId: "123",
+ runAttempt: 1,
+ });
+
+test("builds records and turns missing legs and evals into missing records", (t) => {
+ const records = build(legsDirectory(t));
+ assert.deepEqual(
+ records.map((r) => [r.model_id, r.eval_id, r.missing, r.artifact_path]),
+ [
+ ["acme/alpha", "static-tools/ends-turn-function", null, "raw/agent-tools/alpha/local/1"],
+ ["acme/alpha", "static-tools/no-reply", "eval-missing", "raw/agent-tools/alpha/local/1"],
+ ["mock", "static-tools/ends-turn-function", "job-missing", null],
+ ["mock", "static-tools/no-reply", "job-missing", null],
+ ],
+ );
+ const [completed] = records;
+ assert.equal(completed.eve_version, "0.71.2");
+ assert.equal(completed.model_release, "2026-09-22");
+ assert.equal(completed.verdict, "passed");
+ assert.equal(completed.duration_ms, 1500);
+ assert.deepEqual(completed.assertions, [
+ { name: "replied", severity: "gate", score: 1, passed: true },
+ ]);
+ assert.equal(records[2].model_release, null);
+ assert.equal(records[2].duration_ms, null);
+});
+
+test("excludes live-leg evals whose model steps all ran on another model", (t) => {
+ const legsDir = legsDirectory(t);
+ const leg = join(legsDir, "benchmark-leg-live__agent-tools__alpha__local__1");
+ // Some fixtures script delegation evals with eve's mock models even on live legs.
+ writeEval(leg, "static-tools/no-reply", "eve-mock/notebook-parent");
+ const { records, excluded } = buildWithExclusions(legsDir);
+ assert.deepEqual(excluded, [
+ { leg_id: "live__agent-tools__alpha__local__1", eval_id: "static-tools/no-reply" },
+ ]);
+ const live = records.filter((r) => r.model_id === "acme/alpha");
+ assert.deepEqual(
+ live.map((r) => r.eval_id),
+ ["static-tools/ends-turn-function"],
+ "an excluded eval is neither a result nor missing",
+ );
+});
+
+test("rejects artifacts that were not planned or ran at another commit", (t) => {
+ assert.throws(
+ () => readLegs(plan, legsDirectory(t, { sha: "b".repeat(40) })),
+ /ran at b+, but the plan is for a+/,
+ );
+ const unplanned = { ...plan, legs: plan.legs.slice(1) };
+ assert.throws(() => readLegs(unplanned, legsDirectory(t)), /was not planned/);
+});
+
+function memoryStore() {
+ const objects = new Map();
+ const writes = [];
+ return {
+ objects,
+ writes,
+ async put(pathname, body) {
+ if (objects.has(pathname))
+ throw new Error(`Vercel Blob: This blob already exists (${pathname}).`);
+ objects.set(pathname, Buffer.from(body));
+ writes.push(pathname);
+ },
+ async read(pathname) {
+ return objects.get(pathname) ?? null;
+ },
+ };
+}
+
+test("publishes raw trees and records before the manifest, idempotently and immutably", async (t) => {
+ const legsDir = legsDirectory(t);
+ const legs = readLegs(plan, legsDir);
+ const recordsJsonl = build(legsDir)
+ .map((record) => `${JSON.stringify(record)}\n`)
+ .join("");
+ const store = memoryStore();
+ const input = {
+ plan,
+ legs,
+ recordsJsonl,
+ githubRunId: "123",
+ runAttempt: 1,
+ eveVersion: "0.71.2",
+ store,
+ };
+
+ const result = await publishRun(input);
+ assert.equal(result.prefix, "runs/123/1");
+ assert.equal(store.writes.at(-1), "runs/123/1/manifest.json");
+ assert(
+ store.objects.has(
+ "runs/123/1/raw/agent-tools/alpha/local/1/run/evals/static-tools/ends-turn-function.json",
+ ),
+ );
+ assert(store.objects.has("runs/123/1/raw/agent-tools/alpha/local/1/meta.json"));
+ const manifest = JSON.parse(store.objects.get("runs/123/1/manifest.json"));
+ assert.equal(manifest.record_count, 4);
+ assert.deepEqual(
+ manifest.legs.map((leg) => leg.uploaded),
+ [true, false],
+ );
+
+ const writes = store.writes.length;
+ await publishRun(input);
+ assert.equal(store.writes.length, writes, "identical re-run is a no-op");
+
+ await assert.rejects(
+ publishRun({ ...input, recordsJsonl: recordsJsonl.split("\n").slice(1).join("\n") }),
+ /records.jsonl was already published with different content/,
+ );
+});