The Open Framework for Building Agents
ci: publish nightly model benchmark results from main (#5189)
prhabfb293cf92a0
diff --git a/.github/scripts/discover-e2e-fixtures.mjs b/.github/scripts/discover-e2e-fixtures.mjsindex 31c98af3c..ff718cb8c 100644--- a/.github/scripts/discover-e2e-fixtures.mjs+++ b/.github/scripts/discover-e2e-fixtures.mjs@@ -146,7 +146,7 @@ if (process.argv[1] === fileURLToPath(import.meta.url)) {else process.stdout.write(result.lines);}-function discoverEvalIds(root) {+export function discoverEvalIds(root) {const evalIds = [];const visit = (directory) => {for (const entry of readdirSync(directory, { withFileTypes: true }).sort((a, b) =>diff --git a/.github/scripts/stage-benchmark-leg.sh b/.github/scripts/stage-benchmark-leg.shnew file mode 100755index 000000000..6a23321f8--- /dev/null+++ b/.github/scripts/stage-benchmark-leg.sh@@ -0,0 +1,35 @@+#!/usr/bin/env bash+# Stages one eval-benchmark leg for upload as $RUNNER_TEMP/leg:+# meta.json leg identity and runner context+# expected.json `eve eval --list --json` output, when the run got that far+# run/ the leg's single `.eve/evals/<timestamp>` tree, when one exists+# The publish job (scripts/eval-benchmark/records.mjs) reads this layout.+set -euo pipefail++leg="$RUNNER_TEMP/leg"+rm -rf "$leg"+mkdir -p "$leg"++jq -n \+ --arg leg_id "$LEG_ID" \+ --arg sha "$PLANNED_SHA" \+ --arg model_id "$MODEL_ID" \+ --arg world "$WORLD" \+ --argjson attempt "$ATTEMPT" \+ --arg runner_os "$RUNNER_OS_NAME" \+ '{leg_id: $leg_id, sha: $sha, model_id: $model_id, world: $world, attempt: $attempt, runner_os: $runner_os}' \+ >"$leg/meta.json"++if [ -s "$RUNNER_TEMP/expected.json" ]; then+ cp "$RUNNER_TEMP/expected.json" "$leg/expected.json"+fi++shopt -s nullglob+runs=("$FIXTURE_DIR"/.eve/evals/*/)+if [ "${#runs[@]}" -gt 1 ]; then+ echo "::error::Expected one .eve/evals tree in $FIXTURE_DIR, found ${#runs[@]}."+ exit 1+fi+if [ "${#runs[@]}" -eq 1 ]; then+ cp -R "${runs[0]}" "$leg/run"+fidiff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 0f649d61c..1ab75cdc2 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -101,6 +101,9 @@ jobs:- name: Test workflow and guard helpersrun: node --test .github/scripts/*.test.mjs scripts/hitl-state-readers.test.mjs+ - name: Test eval benchmark scripts+ run: pnpm test:eval-benchmark+test-integration:name: test-integration (${{ matrix.os }})runs-on: ${{ matrix.os }}diff --git a/.github/workflows/eval-benchmark.yml b/.github/workflows/eval-benchmark.ymlnew file mode 100644index 000000000..60cbab1fd--- /dev/null+++ b/.github/workflows/eval-benchmark.yml@@ -0,0 +1,361 @@+name: Eval Benchmark++# Tracks correctness, latency, and cost of fixture evals across models and eve+# versions, and publishes records to private Blob. Runs only on trusted+# `main` revisions and never on pull requests. Run legs hold model credentials+# only; the publish job is the only one that holds the Blob token.+on:+ schedule:+ # Nightly; skipped when main has not moved since the last successful+ # scheduled run. Manual runs may be narrowed, so they never count.+ - cron: "23 7 * * *"+ workflow_dispatch:+ inputs:+ models:+ description: Comma-separated e2e/benchmark.json model names (empty for all, "none" for mock legs only)+ type: string+ default: ""+ fixtures:+ description: Comma-separated e2e/benchmark.json fixture names (empty for all)+ type: string+ default: ""+ attempts:+ description: Attempts per fixture and model (empty for the registry default)+ type: string+ default: ""++concurrency:+ group: eval-benchmark-main+ cancel-in-progress: false++permissions:+ contents: read++jobs:+ plan:+ name: plan+ runs-on: ubuntu-latest+ permissions:+ contents: read+ actions: read+ outputs:+ run: ${{ steps.skip.outputs.run }}+ sha: ${{ steps.skip.outputs.sha }}+ live_matrix: ${{ steps.plan.outputs.live_matrix }}+ mock_matrix: ${{ steps.plan.outputs.mock_matrix }}+ max_parallel: ${{ steps.plan.outputs.max_parallel }}+ evals_per_leg: ${{ steps.plan.outputs.evals_per_leg }}+ steps:+ - name: Require main+ if: github.ref != 'refs/heads/main'+ run: |+ echo "::error::Benchmark results publish only from refs/heads/main (got ${GITHUB_REF})."+ exit 1++ - name: Checkout code+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ persist-credentials: false++ - name: Setup Node.js+ uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0+ with:+ node-version-file: .nvmrc++ - name: Skip an unchanged nightly run+ id: skip+ env:+ GH_TOKEN: ${{ github.token }}+ EVENT_NAME: ${{ github.event_name }}+ run: |+ echo "sha=${GITHUB_SHA}" >> "$GITHUB_OUTPUT"+ if [ "$EVENT_NAME" = "schedule" ]; then+ last_sha="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow eval-benchmark.yml \+ --branch main --event schedule --status success --limit 1 --json headSha --jq '.[0].headSha // ""')"+ if [ "$last_sha" = "$GITHUB_SHA" ]; then+ echo "main is still at the last benchmarked commit ${GITHUB_SHA}; skipping."+ echo "run=false" >> "$GITHUB_OUTPUT"+ exit 0+ fi+ fi+ echo "run=true" >> "$GITHUB_OUTPUT"++ - name: Plan legs+ id: plan+ if: steps.skip.outputs.run == 'true'+ env:+ BENCHMARK_MODELS: ${{ inputs.models }}+ BENCHMARK_FIXTURES: ${{ inputs.fixtures }}+ BENCHMARK_ATTEMPTS: ${{ inputs.attempts }}+ run: node scripts/eval-benchmark/plan.mjs "$RUNNER_TEMP/plan.json" "$GITHUB_SHA"++ - name: Upload plan+ if: steps.skip.outputs.run == 'true'+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1+ with:+ name: benchmark-plan+ path: ${{ runner.temp }}/plan.json+ if-no-files-found: error+ retention-days: 1++ # Live-model legs: fixture × model × attempt on the local world. Eval+ # failures are data, so neither the eval run nor the job fails the workflow.+ # e2e/benchmark.json `concurrency` caps legs (gateway load) and evals per+ # leg (runner contention, which skews latency).+ run-live:+ name: run-live (${{ matrix.fixture }}, ${{ matrix.model_id }}, ${{ matrix.attempt }})+ needs: plan+ if: needs.plan.outputs.run == 'true' && needs.plan.outputs.live_matrix != '[]'+ runs-on: ubuntu-latest+ continue-on-error: true+ timeout-minutes: 90+ strategy:+ fail-fast: false+ max-parallel: ${{ fromJSON(needs.plan.outputs.max_parallel) }}+ matrix:+ include: ${{ fromJSON(needs.plan.outputs.live_matrix) }}+ env:+ AI_GATEWAY_API_KEY: ${{ secrets.AI_GATEWAY_API_KEY }}+ EVE_E2E_MODEL: ${{ matrix.model_id }}+ EVALS_PER_LEG: ${{ needs.plan.outputs.evals_per_leg }}+ EVE_SANDBOX_IMAGE_TAG: latest+ steps:+ - name: Checkout planned commit+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ ref: ${{ needs.plan.outputs.sha }}+ persist-credentials: false++ - name: Setup pnpm+ uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10++ - name: Setup Node.js+ uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0+ with:+ node-version-file: .nvmrc+ cache: "pnpm"++ - name: Install dependencies+ run: pnpm install --frozen-lockfile++ - name: Build eve package+ run: pnpm --filter eve run build++ - name: Build extension fixtures+ # Mirrors e2e-local: extension fixtures ship no committed dist/.+ run: |+ pnpm --filter "./e2e/fixtures/*-extension" run --if-present build+ pnpm --filter dist-extensions update gizmo-extension gadget-extension++ - name: Run fixture evals+ working-directory: ${{ matrix.dir }}+ run: |+ pnpm run --if-present e2e:prepare+ pnpm exec eve eval --list --json > "$RUNNER_TEMP/expected.json"+ if ! pnpm exec eve eval --verbose --max-concurrency "$EVALS_PER_LEG"; then+ echo "::warning::Some evals failed; failures are recorded as benchmark data."+ fi++ - name: Stage leg artifact+ if: always()+ env:+ LEG_ID: ${{ matrix.leg_id }}+ FIXTURE_DIR: ${{ matrix.dir }}+ PLANNED_SHA: ${{ needs.plan.outputs.sha }}+ MODEL_ID: ${{ matrix.model_id }}+ WORLD: ${{ matrix.world }}+ ATTEMPT: ${{ matrix.attempt }}+ RUNNER_OS_NAME: ${{ runner.os }}+ run: bash .github/scripts/stage-benchmark-leg.sh++ - name: Upload leg artifact+ if: always()+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1+ with:+ name: benchmark-leg-${{ matrix.leg_id }}+ path: ${{ runner.temp }}/leg+ if-no-files-found: error+ retention-days: 1++ # Mock-model legs: one per fixture and workflow world. With no model latency+ # or cost, their timings are close to pure eve overhead.+ run-mock:+ name: run-mock (${{ matrix.fixture }}, ${{ matrix.world }})+ needs: plan+ if: needs.plan.outputs.run == 'true' && needs.plan.outputs.mock_matrix != '[]'+ runs-on: ubuntu-latest+ continue-on-error: true+ timeout-minutes: 60+ strategy:+ fail-fast: false+ matrix:+ include: ${{ fromJSON(needs.plan.outputs.mock_matrix) }}+ # Unused by local-world legs; a service container cannot vary per leg.+ services:+ postgres:+ image: postgres:18-alpine+ env:+ POSTGRES_DB: world+ POSTGRES_PASSWORD: world+ POSTGRES_USER: world+ ports:+ - 5432:5432+ options: >-+ --health-cmd "pg_isready --username=world --dbname=world"+ --health-interval 5s+ --health-timeout 5s+ --health-retries 12+ env:+ # Mock models never reach the gateway.+ AI_GATEWAY_API_KEY: invalid-e2e-world-suite-key+ EVE_E2E_MODEL: mock+ EVALS_PER_LEG: ${{ needs.plan.outputs.evals_per_leg }}+ EVE_SANDBOX_IMAGE_TAG: latest+ WORKFLOW_POSTGRES_MAX_POOL_SIZE: "52"+ WORKFLOW_POSTGRES_URL: postgres://world:[email protected]:5432/world+ WORKFLOW_POSTGRES_WORKER_CONCURRENCY: "50"+ steps:+ - name: Checkout planned commit+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ ref: ${{ needs.plan.outputs.sha }}+ persist-credentials: false++ - name: Setup pnpm+ uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10++ - name: Setup Node.js+ uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0+ with:+ node-version-file: .nvmrc+ cache: "pnpm"++ - name: Install dependencies+ run: pnpm install --frozen-lockfile++ - name: Build eve package+ run: pnpm --filter eve run build++ - name: Build extension fixtures+ run: |+ pnpm --filter "./e2e/fixtures/*-extension" run --if-present build+ pnpm --filter dist-extensions update gizmo-extension gadget-extension++ - name: Run fixture evals on the local world+ if: matrix.world == 'local'+ working-directory: ${{ matrix.dir }}+ run: |+ pnpm run --if-present e2e:prepare+ pnpm exec eve eval --list --json --exclude-tag real-model > "$RUNNER_TEMP/expected.json"+ if ! pnpm exec eve eval --verbose --exclude-tag real-model --max-concurrency "$EVALS_PER_LEG"; then+ echo "::warning::Some evals failed; failures are recorded as benchmark data."+ fi++ - name: Run fixture evals on a production server+ if: matrix.world != 'local'+ working-directory: ${{ matrix.dir }}+ run: |+ set -euo pipefail+ pnpm run --if-present e2e:prepare+ pnpm exec eve eval --list --json --exclude-tag real-model > "$RUNNER_TEMP/expected.json"+ if [ "$WORLD" = "postgres" ]; then pnpm exec bootstrap; fi+ pnpm exec eve build+ server_log="$RUNNER_TEMP/server.log"+ EVE_DEV=1 pnpm exec eve start --host 127.0.0.1 --port 3000 >"$server_log" 2>&1 &+ server_pid=$!+ trap 'kill "$server_pid" 2>/dev/null; wait "$server_pid" 2>/dev/null || true' EXIT+ ready=false+ for _ in {1..120}; do+ if curl --fail --silent http://127.0.0.1:3000/eve/v1/health >/dev/null; then+ ready=true+ break+ fi+ if ! kill -0 "$server_pid" 2>/dev/null; then cat "$server_log"; exit 1; fi+ sleep 0.5+ done+ if [ "$ready" != "true" ]; then+ cat "$server_log"+ echo "::error::Timed out waiting for the eve server."+ exit 1+ fi+ if ! pnpm exec eve eval --verbose --exclude-tag real-model --max-concurrency "$EVALS_PER_LEG" \+ --url http://127.0.0.1:3000; then+ echo "::warning::Some evals failed; failures are recorded as benchmark data."+ fi+ env:+ WORLD: ${{ matrix.world }}+ # Only non-local legs select a world; fixtures read an empty value+ # as a package name, so local legs must leave it unset.+ EVE_E2E_WORKFLOW_WORLD: ${{ matrix.world_package }}++ - name: Stage leg artifact+ if: always()+ env:+ LEG_ID: ${{ matrix.leg_id }}+ FIXTURE_DIR: ${{ matrix.dir }}+ PLANNED_SHA: ${{ needs.plan.outputs.sha }}+ MODEL_ID: ${{ matrix.model_id }}+ WORLD: ${{ matrix.world }}+ ATTEMPT: ${{ matrix.attempt }}+ RUNNER_OS_NAME: ${{ runner.os }}+ run: bash .github/scripts/stage-benchmark-leg.sh++ - name: Upload leg artifact+ if: always()+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1+ with:+ name: benchmark-leg-${{ matrix.leg_id }}+ path: ${{ runner.temp }}/leg+ if-no-files-found: error+ retention-days: 1++ # Runs trusted code from the planned main commit, never code a leg uploaded.+ # Planned legs without an artifact become `missing` records.+ publish:+ name: publish+ needs: [plan, run-live, run-mock]+ if: always() && needs.plan.result == 'success' && needs.plan.outputs.run == 'true'+ runs-on: ubuntu-latest+ steps:+ - name: Checkout planned commit+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ ref: ${{ needs.plan.outputs.sha }}+ persist-credentials: false++ - name: Setup pnpm+ uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10++ - name: Setup Node.js+ uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0+ with:+ node-version-file: .nvmrc+ cache: "pnpm"++ - name: Install dependencies+ run: pnpm install --frozen-lockfile++ - name: Download plan+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1+ with:+ name: benchmark-plan+ path: ${{ runner.temp }}/plan++ - name: Prepare leg directory+ run: mkdir -p "$RUNNER_TEMP/legs"++ - name: Download leg artifacts+ # No match is valid (every leg failed to upload); records mark those legs missing.+ continue-on-error: true+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1+ with:+ pattern: benchmark-leg-*+ path: ${{ runner.temp }}/legs++ - name: Derive records+ run: node scripts/eval-benchmark/records.mjs "$RUNNER_TEMP/plan/plan.json" "$RUNNER_TEMP/legs" "$RUNNER_TEMP/records.jsonl"++ - name: Publish to Blob+ env:+ EVE_BENCHMARK_BLOB_READ_WRITE_TOKEN: ${{ secrets.EVE_BENCHMARK_BLOB_READ_WRITE_TOKEN }}+ run: node scripts/eval-benchmark/publish.mjs "$RUNNER_TEMP/plan/plan.json" "$RUNNER_TEMP/legs" "$RUNNER_TEMP/records.jsonl"diff --git a/e2e/benchmark.json b/e2e/benchmark.jsonnew file mode 100644index 000000000..db4471134--- /dev/null+++ b/e2e/benchmark.json@@ -0,0 +1,19 @@+{+ "$comment": "Registry for the model benchmark workflow (.github/workflows/eval-benchmark.yml), separate from e2e/matrix.json so adding a benchmark model never adds a CI check. `models` are slots: a stable `name` (`<lab>-high`, `-mid`, and `-low` tiers per major lab where the lab offers a current model for the tier, plus each open-weight family's main model; `openai-low` is eve's default model) and the gateway `id` that currently fills it. Bump a slot's `id` when its lab ships a successor; records keep both, and the pinned release marks the change. Leave out service-tier variants (`-fast`) unless they are eve's default. Each slot runs every listed fixture `attempts` times against the local world; each id must exist in the AI Gateway catalog, which pins its release. `mockWorlds` run each fixture once with mock models (`local`, or a world with a package in e2e/matrix.json). `concurrency.legs` caps live legs running at once (gateway load); `concurrency.evalsPerLeg` is passed to `eve eval --max-concurrency` on every leg, overriding a fixture's own maxConcurrency, so evals do not share a runner. Both change measured latency, and each run's manifest records them.",+ "models": [+ { "name": "openai-high", "id": "openai/gpt-6.1-sol" },+ { "name": "openai-low", "id": "openai/gpt-6-luna-fast" },+ { "name": "anthropic-high", "id": "anthropic/claude-opus-5.5" },+ { "name": "anthropic-mid", "id": "anthropic/claude-sonnet-5.5" },+ { "name": "anthropic-low", "id": "anthropic/claude-haiku-5.5" },+ { "name": "google-mid", "id": "google/gemini-3.8-flash" },+ { "name": "google-low", "id": "google/gemini-3.5-flash-lite" },+ { "name": "open-kimi", "id": "moonshotai/kimi-k3" },+ { "name": "open-glm", "id": "zai/glm-5.3" },+ { "name": "open-minimax", "id": "minimax/minimax-m3" }+ ],+ "fixtures": ["agent-tools", "agent-subagents", "agent-tasks"],+ "mockWorlds": ["local", "postgres"],+ "attempts": 3,+ "concurrency": { "legs": 8, "evalsPerLeg": 1 }+}diff --git a/package.json b/package.jsonindex 58ac7b840..ebbad9aa3 100644--- a/package.json+++ b/package.json@@ -26,6 +26,7 @@"test": "pnpm test:unit && pnpm test:integration","test:e2e": "turbo run test:e2e --concurrency=1","test:webchat": "pnpm --filter eve run test:webchat",+ "test:eval-benchmark": "node --test scripts/eval-benchmark/*.test.mjs","test:framework-fixtures": "pnpm --filter eve run test:framework-fixtures","test:integration": "turbo run test:integration --filter=eve && turbo run test:integration --filter=\"!eve\"","test:scenario": "turbo run test:scenario",@@ -39,6 +40,7 @@"@changesets/cli": "3.0.1","@microsoft/api-extractor": "7.59.0","@types/node": "catalog:",+ "@vercel/blob": "2.8.0","js-yaml": "4.3.2","oxfmt": "0.66.0","oxlint": "1.81.0",diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 15b20794c..f2a63c2b6 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -293,6 +293,9 @@ importers:'@types/node':specifier: 'catalog:'version: 24.13.3+ '@vercel/blob':+ specifier: 2.8.0+ version: 2.8.0js-yaml:specifier: 4.3.2version: 4.3.2diff --git a/scripts/eval-benchmark/catalog.mjs b/scripts/eval-benchmark/catalog.mjsnew file mode 100644index 000000000..ad6a10a2e--- /dev/null+++ b/scripts/eval-benchmark/catalog.mjs@@ -0,0 +1,37 @@+// Pins benchmark model ids to their AI Gateway catalog release, so a provider+// re-pointing an alias shows up as an identity break instead of a trend.++export const AI_GATEWAY_MODELS_URL = "https://ai-gateway.vercel.sh/v1/models";++/**+ * @param {{ name: string, id: string }[]} models+ * @param {typeof fetch} [fetchImpl]+ * @returns {Promise<{ name: string, id: string, release: string }[]>}+ */+export async function resolveModelReleases(models, fetchImpl = fetch) {+ const response = await fetchImpl(AI_GATEWAY_MODELS_URL, { signal: AbortSignal.timeout(30_000) });+ if (!response.ok) {+ throw new Error(+ `AI Gateway model catalog request failed (${response.status}); cannot pin model releases.`,+ );+ }+ const body = await response.json();+ if (!Array.isArray(body?.data)) throw new Error("AI Gateway returned an invalid model catalog.");+ const catalog = new Map(body.data.map((entry) => [entry?.id, entry]));++ return models.map(({ name, id }) => {+ const entry = catalog.get(id);+ if (entry === undefined) {+ throw new Error(+ `AI Gateway catalog has no model "${id}" (e2e/benchmark.json model "${name}"). Update the id or remove the entry.`,+ );+ }+ // `released` is the catalog's release date in Unix seconds.+ if (typeof entry.released !== "number" || !Number.isFinite(entry.released)) {+ throw new Error(+ `AI Gateway catalog entry "${id}" (e2e/benchmark.json model "${name}") has no release date to pin.`,+ );+ }+ return { name, id, release: new Date(entry.released * 1000).toISOString().slice(0, 10) };+ });+}diff --git a/scripts/eval-benchmark/digest.mjs b/scripts/eval-benchmark/digest.mjsnew file mode 100644index 000000000..e85d9e784--- /dev/null+++ b/scripts/eval-benchmark/digest.mjs@@ -0,0 +1,19 @@+// `fixture_digest`: sha256 over the git tree ids of the fixture directory and+// the shared e2e config at the planned commit. Hashing whole trees cannot miss+// a helper an eval imports, at the cost of per-eval granularity: any edit to a+// fixture breaks comparability for all of its evals. eve itself is excluded,+// since eve's version is what benchmarks compare.+import { execFileSync } from "node:child_process";+import { createHash } from "node:crypto";++const SHARED_PATHS = ["e2e/fixtures/e2e-config"];++/** @returns {string} hex digest */+export function fixtureDigest(sha, fixtureDir) {+ const hash = createHash("sha256");+ for (const path of [fixtureDir, ...SHARED_PATHS]) {+ const tree = execFileSync("git", ["rev-parse", `${sha}:${path}`], { encoding: "utf8" }).trim();+ hash.update(`${path}\0${tree}\n`);+ }+ return hash.digest("hex");+}diff --git a/scripts/eval-benchmark/plan.mjs b/scripts/eval-benchmark/plan.mjsnew file mode 100644index 000000000..f54390b9a--- /dev/null+++ b/scripts/eval-benchmark/plan.mjs@@ -0,0 +1,278 @@+// Plans one benchmark run from e2e/benchmark.json:+//+// live_matrix attempt × fixture × model legs on the local world, ordered so+// the legs running together spread across models+// max_parallel, evals_per_leg+// the registry's concurrency caps, which bound gateway load+// and keep evals from sharing a runner+// mock_matrix fixture × mock world legs, one attempt each+// plan.json the resolved registry snapshot (models pinned to their AI+// Gateway release, fixture digests, expected eval ids) plus+// every leg,+// handed to the publish job+//+// Dispatch inputs narrow the run through BENCHMARK_MODELS and+// BENCHMARK_FIXTURES (comma-separated registry names) and BENCHMARK_ATTEMPTS.+//+// Usage: node scripts/eval-benchmark/plan.mjs <plan.json> [sha]+import { execFileSync } from "node:child_process";+import { appendFileSync, existsSync, readFileSync, writeFileSync } from "node:fs";+import { join } from "node:path";+import { fileURLToPath } from "node:url";++import { discoverEvalIds } from "../../.github/scripts/discover-e2e-fixtures.mjs";+import { resolveModelReleases } from "./catalog.mjs";+import { fixtureDigest } from "./digest.mjs";++export const PLAN_SCHEMA_VERSION = 1;+const FIXTURE_ROOTS = ["e2e/fixtures", "apps/fixtures"];+const NAME = /^[a-z0-9-]+$/;+const MAX_ATTEMPTS = 10;+const MAX_PARALLEL_LEGS = 50;+const MAX_EVALS_PER_LEG = 16;++/**+ * Validate the registry and apply dispatch narrowing. Pure apart from the+ * injected fixture lookup, so it is testable without a checkout.+ *+ * @param {{+ * registry: any,+ * matrix: any,+ * findFixture: (name: string) => { dir: string, evalIds: string[], packageJson?: any } | undefined,+ * narrow?: { models?: string, fixtures?: string, attempts?: string },+ * }} input+ */+export function selectBenchmark({ registry, matrix, findFixture, narrow = {} }) {+ const models = namedEntries(registry?.models, "models");+ const attempts = positiveInteger(registry?.attempts, 'e2e/benchmark.json: "attempts"');+ const concurrency = {+ legs: boundedInteger(+ registry?.concurrency?.legs,+ MAX_PARALLEL_LEGS,+ 'e2e/benchmark.json: "concurrency.legs"',+ ),+ evalsPerLeg: boundedInteger(+ registry?.concurrency?.evalsPerLeg,+ MAX_EVALS_PER_LEG,+ 'e2e/benchmark.json: "concurrency.evalsPerLeg"',+ ),+ };+ const fixtureNames = names(registry?.fixtures, "fixtures");+ const mockWorlds = names(registry?.mockWorlds ?? [], "mockWorlds", { allowEmpty: true });+ const matrixWorlds = new Map((matrix?.worlds ?? []).map((world) => [world.name, world]));+ for (const world of mockWorlds) {+ // Mock legs boot a local server, so a non-local world must be a package+ // the fixture can select (EVE_E2E_WORKFLOW_WORLD), not a deploy target.+ if (world !== "local" && matrixWorlds.get(world)?.package === undefined) {+ const packaged = [...matrixWorlds.values()].filter((w) => w.package).map((w) => w.name);+ throw new Error(+ `e2e/benchmark.json: mock world "${world}" must be "local" or a world with a package in e2e/matrix.json (${packaged.join(", ")}).`,+ );+ }+ }++ const fixtures = fixtureNames.map((name) => {+ const found = findFixture(name);+ if (found === undefined) {+ throw new Error(+ `e2e/benchmark.json: fixture "${name}" was not found with an evals/ directory under ${FIXTURE_ROOTS.join(" or ")}.`,+ );+ }+ if (found.evalIds.length === 0)+ throw new Error(`e2e/benchmark.json: fixture "${name}" has no *.eval.ts files.`);+ return { name, ...found };+ });++ const selectedModels = narrowNames(models, narrow.models, "model");+ const selectedFixtures = narrowNames(fixtures, narrow.fixtures, "fixture");+ const selectedAttempts =+ narrow.attempts === undefined || narrow.attempts === ""+ ? attempts+ : positiveInteger(Number(narrow.attempts), "attempts input");++ return {+ attempts: selectedAttempts,+ concurrency,+ models: selectedModels,+ fixtures: selectedFixtures,+ mockWorlds: mockWorlds.map((name) => ({ name, package: matrixWorlds.get(name)?.package })),+ };+}++/** Expand a selection into run legs. Leg ids are stable artifact-name suffixes. */+export function planLegs({ models, fixtures, attempts, mockWorlds }) {+ // GitHub starts matrix legs in order and cannot cap concurrency per model,+ // so model-innermost order keeps each wave of legs spread across models and+ // each eval's attempts spread over time.+ const live = [];+ for (let attempt = 1; attempt <= attempts; attempt += 1) {+ for (const fixture of fixtures) {+ for (const model of models) {+ live.push(+ leg({ kind: "live", fixture, model_name: model.name, model_id: model.id, attempt }),+ );+ }+ }+ }+ const mock = [];+ for (const fixture of fixtures) {+ const selectedWorlds = fixture.packageJson?.e2e?.worlds;+ for (const world of mockWorlds) {+ // `local` is the default world every fixture boots on; other worlds+ // follow the fixture's own e2e.worlds selection.+ if (+ world.name !== "local" &&+ Array.isArray(selectedWorlds) &&+ !selectedWorlds.includes(world.name)+ )+ continue;+ mock.push(+ leg({ kind: "mock", fixture, model_name: "mock", model_id: "mock", world, attempt: 1 }),+ );+ }+ }+ return { live, mock };+}++function leg({ kind, fixture, model_name, model_id, world = { name: "local" }, attempt }) {+ return {+ // Registry names never contain "_", so "__" keeps ids unambiguous.+ leg_id: [kind, fixture.name, model_name, world.name, attempt].join("__"),+ kind,+ fixture: fixture.name,+ dir: fixture.dir,+ model_name,+ model_id,+ world: world.name,+ world_package: world.package ?? "",+ attempt,+ };+}++function namedEntries(entries, key) {+ if (!Array.isArray(entries) || entries.length === 0)+ throw new Error(`e2e/benchmark.json: "${key}" must be a non-empty array.`);+ const seen = new Set();+ for (const entry of entries) {+ if (typeof entry?.name !== "string" || !NAME.test(entry.name))+ throw new Error(`e2e/benchmark.json: every "${key}" entry needs a lowercase-dashed "name".`);+ if (typeof entry.id !== "string" || entry.id.length === 0)+ throw new Error(`e2e/benchmark.json: "${key}" entry "${entry.name}" needs a gateway "id".`);+ if (seen.has(entry.name) || seen.has(entry.id))+ throw new Error(`e2e/benchmark.json: duplicate "${key}" entry "${entry.name}".`);+ seen.add(entry.name).add(entry.id);+ }+ return entries.map(({ name, id }) => ({ name, id }));+}++function names(entries, key, { allowEmpty = false } = {}) {+ if (!Array.isArray(entries) || (!allowEmpty && entries.length === 0))+ throw new Error(`e2e/benchmark.json: "${key}" must be a non-empty array.`);+ if (entries.some((entry) => typeof entry !== "string" || !NAME.test(entry)))+ throw new Error(+ `e2e/benchmark.json: "${key}" entries must be lowercase alphanumerics and dashes.`,+ );+ if (new Set(entries).size !== entries.length)+ throw new Error(`e2e/benchmark.json: "${key}" contains duplicates.`);+ return entries;+}++function positiveInteger(value, label) {+ return boundedInteger(value, MAX_ATTEMPTS, label);+}++function boundedInteger(value, max, label) {+ if (!Number.isInteger(value) || value < 1 || value > max)+ throw new Error(`${label} must be an integer from 1 to ${max}.`);+ return value;+}++/** An empty input keeps every entry; otherwise every requested name must exist. */+function narrowNames(entries, input, label) {+ if (input === undefined || input.trim() === "") return entries;+ const requested = input+ .split(",")+ .map((name) => name.trim())+ .filter(Boolean);+ if (requested.length === 1 && requested[0] === "none") return [];+ if (new Set(requested).size !== requested.length)+ throw new Error(`Benchmark ${label} input "${input}" contains duplicates.`);+ const known = new Map(entries.map((entry) => [entry.name, entry]));+ const unknown = requested.filter((name) => !known.has(name));+ if (unknown.length > 0) {+ throw new Error(+ `Unknown benchmark ${label}${unknown.length === 1 ? "" : "s"} ${unknown.join(", ")}; choose from ${[...known.keys()].join(", ")} (or "none").`,+ );+ }+ return requested.map((name) => known.get(name));+}++function findFixtureOnDisk(name) {+ for (const root of FIXTURE_ROOTS) {+ const dir = join(root, name);+ if (!existsSync(join(dir, "evals"))) continue;+ const packageJsonPath = join(dir, "package.json");+ return {+ dir,+ evalIds: discoverEvalIds(join(dir, "evals")),+ packageJson: existsSync(packageJsonPath)+ ? JSON.parse(readFileSync(packageJsonPath, "utf8"))+ : undefined,+ };+ }+ return undefined;+}++if (process.argv[1] === fileURLToPath(import.meta.url)) {+ const [planPath, shaArgument] = process.argv.slice(2);+ if (planPath === undefined) {+ console.error("Usage: node scripts/eval-benchmark/plan.mjs <plan.json> [sha]");+ process.exit(1);+ }+ const sha =+ shaArgument ?? execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }).trim();+ const selection = selectBenchmark({+ registry: JSON.parse(readFileSync("e2e/benchmark.json", "utf8")),+ matrix: JSON.parse(readFileSync("e2e/matrix.json", "utf8")),+ findFixture: findFixtureOnDisk,+ narrow: {+ models: process.env.BENCHMARK_MODELS,+ fixtures: process.env.BENCHMARK_FIXTURES,+ attempts: process.env.BENCHMARK_ATTEMPTS,+ },+ });+ const models = await resolveModelReleases(selection.models);+ const { live, mock } = planLegs({ ...selection, models });+ const plan = {+ schema_version: PLAN_SCHEMA_VERSION,+ sha,+ planned_at: new Date().toISOString(),+ registry: {+ attempts: selection.attempts,+ // Concurrency changes measured latency, so every manifest records it.+ concurrency: {+ legs: selection.concurrency.legs,+ evals_per_leg: selection.concurrency.evalsPerLeg,+ },+ models,+ mock_worlds: selection.mockWorlds.map((world) => world.name),+ fixtures: selection.fixtures.map(({ name, dir, evalIds }) => ({+ name,+ dir,+ digest: fixtureDigest(sha, dir),+ eval_ids: evalIds,+ })),+ },+ legs: [...live, ...mock],+ };+ writeFileSync(planPath, `${JSON.stringify(plan, null, 2)}\n`);+ const lines = [+ `live_matrix=${JSON.stringify(live)}`,+ `mock_matrix=${JSON.stringify(mock)}`,+ `max_parallel=${selection.concurrency.legs}`,+ `evals_per_leg=${selection.concurrency.evalsPerLeg}`,+ ];+ console.error(`Planned ${live.length} live and ${mock.length} mock legs at ${sha}.`);+ if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, `${lines.join("\n")}\n`);+ else process.stdout.write(`${lines.join("\n")}\n`);+}diff --git a/scripts/eval-benchmark/plan.test.mjs b/scripts/eval-benchmark/plan.test.mjsnew file mode 100644index 000000000..48ef391f6--- /dev/null+++ b/scripts/eval-benchmark/plan.test.mjs@@ -0,0 +1,151 @@+import test from "node:test";+import assert from "node:assert/strict";+import { readFileSync } from "node:fs";+import { resolveModelReleases } from "./catalog.mjs";+import { planLegs, selectBenchmark } from "./plan.mjs";++const registry = {+ models: [+ { name: "alpha", id: "acme/alpha" },+ { name: "beta", id: "acme/beta" },+ ],+ fixtures: ["agent-one", "agent-two"],+ mockWorlds: ["local", "postgres"],+ attempts: 2,+ concurrency: { legs: 3, evalsPerLeg: 1 },+};+const matrix = {+ worlds: [{ name: "vercel" }, { name: "postgres", package: "@workflow/world-postgres" }],+};+const fixtures = {+ "agent-one": { dir: "e2e/fixtures/agent-one", evalIds: ["a", "b"] },+ "agent-two": {+ dir: "e2e/fixtures/agent-two",+ evalIds: ["c"],+ packageJson: { e2e: { worlds: ["vercel"] } },+ },+};+const select = (overrides = {}, narrow = {}) =>+ selectBenchmark({+ registry: { ...registry, ...overrides },+ matrix,+ findFixture: (name) => fixtures[name],+ narrow,+ });++test("expands fixtures × models × attempts and mock worlds the fixture selects", () => {+ const selection = select();+ assert.deepEqual(selection.concurrency, { legs: 3, evalsPerLeg: 1 });+ const { live, mock } = planLegs(selection);+ assert.equal(live.length, 2 * 2 * 2);+ // Models vary fastest so legs started together spread across models, and+ // each eval's attempts are spread over the run.+ assert.deepEqual(+ live.slice(0, 5).map((leg) => leg.leg_id),+ [+ "live__agent-one__alpha__local__1",+ "live__agent-one__beta__local__1",+ "live__agent-two__alpha__local__1",+ "live__agent-two__beta__local__1",+ "live__agent-one__alpha__local__2",+ ],+ );+ assert.deepEqual(live[0], {+ leg_id: "live__agent-one__alpha__local__1",+ kind: "live",+ fixture: "agent-one",+ dir: "e2e/fixtures/agent-one",+ model_name: "alpha",+ model_id: "acme/alpha",+ world: "local",+ world_package: "",+ attempt: 1,+ });+ assert.equal(new Set(live.map((leg) => leg.leg_id)).size, live.length);+ assert.deepEqual(+ mock.map((leg) => [leg.leg_id, leg.world_package]),+ [+ ["mock__agent-one__mock__local__1", ""],+ ["mock__agent-one__mock__postgres__1", "@workflow/world-postgres"],+ ["mock__agent-two__mock__local__1", ""],+ ],+ );+});++test("dispatch inputs narrow every dimension and reject unknown names", () => {+ const selection = select({}, { models: "beta", fixtures: "agent-two", attempts: "1" });+ assert.deepEqual(+ planLegs(selection).live.map((leg) => leg.leg_id),+ ["live__agent-two__beta__local__1"],+ );+ assert.deepEqual(planLegs(select({}, { fixtures: "none" })), { live: [], mock: [] });+ assert.throws(+ () => select({}, { models: "gamma" }),+ /Unknown benchmark model gamma; choose from alpha, beta/,+ );+ assert.throws(+ () => select({}, { models: "beta,beta" }),+ /Benchmark model input "beta,beta" contains duplicates/,+ );+ assert.throws(+ () => select({}, { attempts: "0" }),+ /attempts input must be an integer from 1 to 10/,+ );+});++test("registry errors name the offending entry", () => {+ for (const [overrides, message] of [+ [{ fixtures: ["agent-missing"] }, /fixture "agent-missing" was not found/],+ [{ mockWorlds: ["moon"] }, /mock world "moon" must be "local" or a world with a package/],+ [{ mockWorlds: ["vercel"] }, /mock world "vercel" must be "local" or a world with a package/],+ [+ {+ models: [+ { name: "alpha", id: "x" },+ { name: "alpha", id: "y" },+ ],+ },+ /duplicate "models" entry "alpha"/,+ ],+ [{ concurrency: undefined }, /"concurrency.legs" must be an integer from 1 to 50/],+ [+ { concurrency: { legs: 5, evalsPerLeg: 0 } },+ /"concurrency.evalsPerLeg" must be an integer from 1 to 16/,+ ],+ ])+ assert.throws(() => select(overrides), message);+});++test("the committed registry is valid", () => {+ const committed = JSON.parse(readFileSync(new URL("../../e2e/benchmark.json", import.meta.url)));+ const committedMatrix = JSON.parse(+ readFileSync(new URL("../../e2e/matrix.json", import.meta.url)),+ );+ const selection = selectBenchmark({+ registry: committed,+ matrix: committedMatrix,+ findFixture: () => ({ dir: "fixture", evalIds: ["eval"] }),+ });+ assert.equal(selection.models.length, committed.models.length);+});++const catalog =+ (data, ok = true) =>+ async () => ({ ok, status: ok ? 200 : 503, json: async () => ({ data }) });++test("pins each model to its catalog release and fails before running on unknown ids", async () => {+ const released = Date.UTC(2026, 8, 22) / 1000;+ const entries = [{ id: "acme/alpha", released }];+ assert.deepEqual(+ await resolveModelReleases([{ name: "alpha", id: "acme/alpha" }], catalog(entries)),+ [{ name: "alpha", id: "acme/alpha", release: "2026-09-22" }],+ );+ await assert.rejects(+ resolveModelReleases([{ name: "beta", id: "acme/beta" }], catalog(entries)),+ /AI Gateway catalog has no model "acme\/beta" \(e2e\/benchmark.json model "beta"\)/,+ );+ await assert.rejects(+ resolveModelReleases([{ name: "alpha", id: "acme/alpha" }], catalog([{ id: "acme/alpha" }])),+ /has no release date to pin/,+ );+});diff --git a/scripts/eval-benchmark/publish.mjs b/scripts/eval-benchmark/publish.mjsnew file mode 100644index 000000000..6e6845b75--- /dev/null+++ b/scripts/eval-benchmark/publish.mjs@@ -0,0 +1,171 @@+// Publishes one benchmark run to private Vercel Blob. Every object is+// immutable; re-running with identical content is a no-op and different+// content fails. manifest.json is written last, so its presence tells ingest+// the run is complete:+//+// runs/<github_run_id>/<run_attempt>/raw/<fixture>/<model>/<world>/<attempt>/...+// runs/<github_run_id>/<run_attempt>/records.jsonl+// runs/<github_run_id>/<run_attempt>/manifest.json+//+// Usage: node scripts/eval-benchmark/publish.mjs <plan.json> <legs-dir> <records.jsonl>+// Requires EVE_BENCHMARK_BLOB_READ_WRITE_TOKEN, GITHUB_RUN_ID, GITHUB_RUN_ATTEMPT.+import { createHash } from "node:crypto";+import { readdirSync, readFileSync } from "node:fs";+import { join, relative } from "node:path";+import { fileURLToPath } from "node:url";++import { RECORD_SCHEMA_VERSION, legArtifactPath, readLegs } from "./records.mjs";++const UPLOAD_CONCURRENCY = 8;++/**+ * @typedef {{+ * put: (pathname: string, body: Buffer, contentType: string) => Promise<void>,+ * read: (pathname: string) => Promise<Buffer | null>,+ * }} ImmutableStore `put` must reject with an "already exists" error rather than overwrite.+ */++/**+ * @param {{ plan: any, legs: ReturnType<typeof readLegs>, recordsJsonl: string, githubRunId: string, runAttempt: number, eveVersion: string, store: ImmutableStore }} input+ */+export async function publishRun({+ plan,+ legs,+ recordsJsonl,+ githubRunId,+ runAttempt,+ eveVersion,+ store,+}) {+ const records = recordsJsonl+ .split("\n")+ .filter(Boolean)+ .map((line) => JSON.parse(line));+ for (const record of records) {+ if (record.github_run_id !== String(githubRunId) || record.run_attempt !== runAttempt)+ throw new Error(`Record ${record.fixture}/${record.eval_id} belongs to another run.`);+ }++ const prefix = `runs/${githubRunId}/${runAttempt}`;+ const uploads = legs.flatMap(({ leg, dir }) =>+ dir === undefined+ ? []+ : listFiles(dir).map((path) => ({+ pathname: `${prefix}/${legArtifactPath(leg)}/${relative(dir, path).replaceAll("\\", "/")}`,+ path,+ })),+ );+ await mapLimit(uploads, UPLOAD_CONCURRENCY, ({ pathname, path }) =>+ putImmutable(store, pathname, readFileSync(path), contentTypeOf(path)),+ );++ const recordsBody = Buffer.from(recordsJsonl);+ await putImmutable(store, `${prefix}/records.jsonl`, recordsBody, "application/x-ndjson");+ const manifest = {+ schema_version: RECORD_SCHEMA_VERSION,+ github_run_id: String(githubRunId),+ run_attempt: runAttempt,+ sha: plan.sha,+ eve_version: eveVersion,+ planned_at: plan.planned_at,+ registry: plan.registry,+ legs: legs.map(({ leg, dir }) => ({ ...leg, uploaded: dir !== undefined })),+ record_count: records.length,+ records_sha256: createHash("sha256").update(recordsBody).digest("hex"),+ };+ await putImmutable(+ store,+ `${prefix}/manifest.json`,+ Buffer.from(`${JSON.stringify(manifest, null, 2)}\n`),+ "application/json",+ );+ return { prefix, objects: uploads.length + 2, records: records.length };+}++async function putImmutable(store, pathname, body, contentType) {+ try {+ await store.put(pathname, body, contentType);+ } catch (error) {+ if (!(error instanceof Error) || !error.message.includes("already exists")) throw error;+ const published = await store.read(pathname);+ if (published === null) throw new Error(`${pathname} exists but could not be read back.`);+ if (!published.equals(body)) {+ throw new Error(+ `${pathname} was already published with different content; benchmark runs are immutable. Re-run the workflow to get a new run attempt instead.`,+ );+ }+ }+}++function listFiles(dir) {+ return readdirSync(dir, { withFileTypes: true, recursive: true })+ .filter((entry) => entry.isFile())+ .map((entry) => join(entry.parentPath, entry.name))+ .sort();+}++function contentTypeOf(path) {+ if (path.endsWith(".json")) return "application/json";+ if (path.endsWith(".ndjson") || path.endsWith(".jsonl")) return "application/x-ndjson";+ return "application/octet-stream";+}++async function mapLimit(items, limit, task) {+ let next = 0;+ const workers = Array.from({ length: Math.min(limit, items.length) }, async () => {+ while (next < items.length) await task(items[next++]);+ });+ await Promise.all(workers);+}++/** Private, immutable Blob store backed by @vercel/blob. */+export async function vercelBlobStore(token) {+ const { get, put } = await import("@vercel/blob");+ return {+ async put(pathname, body, contentType) {+ await put(pathname, body, {+ token,+ access: "private",+ addRandomSuffix: false,+ allowOverwrite: false,+ cacheControlMaxAge: 31_536_000,+ contentType,+ });+ },+ async read(pathname) {+ const result = await get(pathname, { token, access: "private", useCache: false });+ return result === null ? null : Buffer.from(await new Response(result.stream).arrayBuffer());+ },+ };+}++if (process.argv[1] === fileURLToPath(import.meta.url)) {+ const [planPath, legsDir, recordsPath] = process.argv.slice(2);+ if (recordsPath === undefined) {+ console.error(+ "Usage: node scripts/eval-benchmark/publish.mjs <plan.json> <legs-dir> <records.jsonl>",+ );+ process.exit(1);+ }+ const token = process.env.EVE_BENCHMARK_BLOB_READ_WRITE_TOKEN;+ if (!token)+ throw new Error(+ "EVE_BENCHMARK_BLOB_READ_WRITE_TOKEN is required to publish benchmark results.",+ );+ const githubRunId = process.env.GITHUB_RUN_ID;+ if (githubRunId === undefined)+ throw new Error("GITHUB_RUN_ID is required to key the published run.");+ const plan = JSON.parse(readFileSync(planPath, "utf8"));+ const result = await publishRun({+ plan,+ legs: readLegs(plan, legsDir),+ recordsJsonl: readFileSync(recordsPath, "utf8"),+ githubRunId,+ runAttempt: Number(process.env.GITHUB_RUN_ATTEMPT ?? "1"),+ eveVersion: JSON.parse(readFileSync("packages/eve/package.json", "utf8")).version,+ store: await vercelBlobStore(token),+ });+ console.error(+ `Published ${result.records} records and ${result.objects} objects to ${result.prefix}.`,+ );+}diff --git a/scripts/eval-benchmark/records.mjs b/scripts/eval-benchmark/records.mjsnew file mode 100644index 000000000..12a27d5c6--- /dev/null+++ b/scripts/eval-benchmark/records.mjs@@ -0,0 +1,191 @@+// Derives records.jsonl from the run legs' uploaded artifacts. Each leg+// artifact (downloaded to <legs-dir>/benchmark-leg-<leg_id>/) holds:+//+// meta.json { leg_id, sha, runner_os } written by the run job+// expected.json `eve eval --list --json` under the leg's tag filters+// run/ the leg's single `.eve/evals/<timestamp>` tree+//+// Each record is one attempt of one eval on one model and world, keyed by+// (github_run_id, run_attempt, fixture, eval_id, model_id, world, attempt).+// Results are comparable across runs when (eval_id, fixture_digest, model_id,+// model_release, world) match. Planned legs that uploaded nothing, and expected+// evals with no result, become `missing` records so coverage stays honest.+// Richer measurements can be derived later from the published raw trees.+//+// A live leg's eval whose model steps all ran on another model (fixtures script+// some delegation evals with eve's mock models) says nothing about the leg's+// model, so it is excluded; the mock track still measures it.+//+// Usage: node scripts/eval-benchmark/records.mjs <plan.json> <legs-dir> <records.jsonl>+import { existsSync, readdirSync, readFileSync, writeFileSync } from "node:fs";+import { join } from "node:path";+import { fileURLToPath } from "node:url";++export const RECORD_SCHEMA_VERSION = 1;+export const LEG_ARTIFACT_PREFIX = "benchmark-leg-";++/** Map downloaded leg directories onto planned legs, rejecting anything unplanned. */+export function readLegs(plan, legsDir) {+ const planned = new Map(plan.legs.map((leg) => [leg.leg_id, leg]));+ const found = new Map();+ const entries = existsSync(legsDir) ? readdirSync(legsDir, { withFileTypes: true }) : [];+ for (const entry of entries) {+ if (!entry.isDirectory() || !entry.name.startsWith(LEG_ARTIFACT_PREFIX))+ throw new Error(+ `Unexpected entry "${entry.name}" in ${legsDir}; expected ${LEG_ARTIFACT_PREFIX}<leg_id> directories.`,+ );+ const dir = join(legsDir, entry.name);+ const metaPath = join(dir, "meta.json");+ if (!existsSync(metaPath)) throw new Error(`${dir} has no meta.json.`);+ const meta = JSON.parse(readFileSync(metaPath, "utf8"));+ if (`${LEG_ARTIFACT_PREFIX}${meta.leg_id}` !== entry.name)+ throw new Error(`${metaPath}: leg_id "${meta.leg_id}" does not match its artifact name.`);+ if (!planned.has(meta.leg_id))+ throw new Error(`${metaPath}: leg "${meta.leg_id}" was not planned.`);+ if (meta.sha !== plan.sha)+ throw new Error(`${metaPath}: ran at ${meta.sha}, but the plan is for ${plan.sha}.`);+ const expectedPath = join(dir, "expected.json");+ found.set(meta.leg_id, {+ dir,+ meta,+ expected: existsSync(expectedPath)+ ? JSON.parse(readFileSync(expectedPath, "utf8")).map((entry) => entry.id)+ : undefined,+ });+ }+ return plan.legs.map((leg) => ({ leg, ...found.get(leg.leg_id) }));+}++/** Path of a leg's raw tree, relative to the run's Blob prefix. */+export function legArtifactPath(leg) {+ return `raw/${leg.fixture}/${leg.model_name}/${leg.world}/${leg.attempt}`;+}++/**+ * @param {{+ * plan: any,+ * legs: ReturnType<typeof readLegs>,+ * githubRunId: string,+ * runAttempt: number,+ * }} input+ */+export function buildRecords({ plan, legs, githubRunId, runAttempt }) {+ const excluded = [];+ const fixtures = new Map(plan.registry.fixtures.map((fixture) => [fixture.name, fixture]));+ const releases = new Map(plan.registry.models.map((model) => [model.id, model.release]));+ const attempts = [];+ for (const { leg, dir, meta, expected } of legs) {+ const fixture = fixtures.get(leg.fixture);+ const artifacts = dir === undefined ? [] : listEvalArtifacts(join(dir, "run"));+ const results = new Map(+ artifacts.map((path) => {+ const artifact = JSON.parse(readFileSync(path, "utf8"));+ return [artifact.id, artifact];+ }),+ );+ const missingReason = dir === undefined ? "job-missing" : "eval-missing";+ const expectedIds = expected ?? fixture.eval_ids;+ for (const evalId of new Set([...expectedIds, ...results.keys()])) {+ const artifact = results.get(evalId);+ if (leg.kind === "live" && artifact !== undefined && !ranOnModel(artifact, leg.model_id)) {+ excluded.push({ leg_id: leg.leg_id, eval_id: evalId });+ continue;+ }+ attempts.push({+ leg,+ meta,+ evalId,+ artifact,+ missingReason,+ uploaded: dir !== undefined,+ });+ }+ }++ const records = attempts.map(({ leg, meta, evalId, artifact, missingReason, uploaded }) => {+ const startedAt = artifact?.startedAt ?? null;+ const completedAt = artifact?.completedAt ?? null;+ return {+ schema_version: RECORD_SCHEMA_VERSION,+ github_run_id: String(githubRunId),+ run_attempt: runAttempt,+ fixture: leg.fixture,+ eval_id: evalId,+ model_id: leg.model_id,+ model_slot: leg.model_name,+ model_release: leg.kind === "mock" ? null : (releases.get(leg.model_id) ?? null),+ world: leg.world,+ attempt: leg.attempt,+ fixture_digest: fixtures.get(leg.fixture).digest,+ sha: plan.sha,+ eve_version: artifact?.result?.runtimeIdentity?.eveVersion ?? null,+ runner_os: meta?.runner_os ?? null,+ missing: artifact === undefined ? missingReason : null,+ verdict: artifact?.verdict ?? null,+ assertions: (artifact?.assertions ?? []).map(({ name, severity, score, passed }) => ({+ name,+ severity,+ score,+ passed,+ })),+ error: artifact?.error ?? null,+ started_at: startedAt,+ completed_at: completedAt,+ duration_ms:+ startedAt !== null && completedAt !== null+ ? Date.parse(completedAt) - Date.parse(startedAt)+ : null,+ artifact_path: uploaded ? legArtifactPath(leg) : null,+ };+ });+ return { records, excluded };+}++/**+ * False only when the eval made model calls and none used `modelId`. An eval+ * that never reached a model step stays, so its failure is recorded.+ */+function ranOnModel(artifact, modelId) {+ const models = artifact.result?.derived?.models ?? [];+ return models.length === 0 || models.includes(modelId);+}++/** Find every eval detail artifact under one `.eve/evals/<timestamp>` tree. */+export function listEvalArtifacts(runDirectory) {+ const evalsDirectory = join(runDirectory, "evals");+ if (!existsSync(evalsDirectory)) return [];+ return readdirSync(evalsDirectory, { withFileTypes: true, recursive: true })+ .filter((entry) => entry.isFile() && entry.name.endsWith(".json"))+ .map((entry) => join(entry.parentPath, entry.name))+ .sort();+}++if (process.argv[1] === fileURLToPath(import.meta.url)) {+ const [planPath, legsDir, outPath] = process.argv.slice(2);+ if (outPath === undefined) {+ console.error(+ "Usage: node scripts/eval-benchmark/records.mjs <plan.json> <legs-dir> <records.jsonl>",+ );+ process.exit(1);+ }+ const githubRunId = process.env.GITHUB_RUN_ID;+ const runAttempt = Number(process.env.GITHUB_RUN_ATTEMPT ?? "1");+ if (githubRunId === undefined)+ throw new Error("GITHUB_RUN_ID is required to key benchmark records.");+ const plan = JSON.parse(readFileSync(planPath, "utf8"));+ const { records, excluded } = buildRecords({+ plan,+ legs: readLegs(plan, legsDir),+ githubRunId,+ runAttempt,+ });+ writeFileSync(outPath, records.map((record) => `${JSON.stringify(record)}\n`).join(""));+ const missing = records.filter((record) => record.missing !== null).length;+ console.error(`Wrote ${records.length} records (${missing} missing) to ${outPath}.`);+ if (excluded.length > 0) {+ console.error(+ `Excluded ${excluded.length} live-leg evals whose model steps never used the leg's model:`,+ );+ for (const { leg_id, eval_id } of excluded) console.error(` ${leg_id} ${eval_id}`);+ }+}diff --git a/scripts/eval-benchmark/records.test.mjs b/scripts/eval-benchmark/records.test.mjsnew file mode 100644index 000000000..31c725c62--- /dev/null+++ b/scripts/eval-benchmark/records.test.mjs@@ -0,0 +1,205 @@+import test from "node:test";+import assert from "node:assert/strict";+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { publishRun } from "./publish.mjs";+import { buildRecords, readLegs } from "./records.mjs";++const SHA = "a".repeat(40);+const plan = {+ schema_version: 1,+ sha: SHA,+ planned_at: "2026-01-01T00:00:00.000Z",+ registry: {+ attempts: 1,+ models: [{ name: "alpha", id: "acme/alpha", release: "2026-09-22" }],+ mock_worlds: ["local"],+ fixtures: [+ {+ name: "agent-tools",+ dir: "e2e/fixtures/agent-tools",+ digest: "f".repeat(64),+ eval_ids: ["static-tools/ends-turn-function", "static-tools/no-reply"],+ },+ ],+ },+ legs: [+ {+ leg_id: "live__agent-tools__alpha__local__1",+ kind: "live",+ fixture: "agent-tools",+ dir: "e2e/fixtures/agent-tools",+ model_name: "alpha",+ model_id: "acme/alpha",+ world: "local",+ world_package: "",+ attempt: 1,+ },+ {+ leg_id: "mock__agent-tools__mock__local__1",+ kind: "mock",+ fixture: "agent-tools",+ dir: "e2e/fixtures/agent-tools",+ model_name: "mock",+ model_id: "mock",+ world: "local",+ world_package: "",+ attempt: 1,+ },+ ],+};++/** One uploaded live leg whose run produced one of its two expected evals; the mock leg is missing. */+function legsDirectory(t, meta = {}) {+ const root = mkdtempSync(join(tmpdir(), "eve-benchmark-"));+ t.after(() => rmSync(root, { recursive: true, force: true }));+ const leg = join(root, "benchmark-leg-live__agent-tools__alpha__local__1");+ mkdirSync(join(leg, "run", "evals", "static-tools"), { recursive: true });+ writeFileSync(+ join(leg, "meta.json"),+ JSON.stringify({+ leg_id: "live__agent-tools__alpha__local__1",+ sha: SHA,+ runner_os: "Linux",+ ...meta,+ }),+ );+ writeEval(leg, "static-tools/ends-turn-function", "acme/alpha");+ writeFileSync(join(leg, "run", "summary.json"), "{}");+ return root;+}++function writeEval(leg, evalId, modelId) {+ const artifact = {+ id: evalId,+ verdict: "passed",+ assertions: [{ name: "replied", severity: "gate", score: 1, passed: true, extra: "x" }],+ startedAt: "2026-01-01T00:00:00.000Z",+ completedAt: "2026-01-01T00:00:01.500Z",+ result: {+ runtimeIdentity: { eveVersion: "0.71.2" },+ derived: { models: [modelId] },+ },+ };+ writeFileSync(join(leg, "run", "evals", `${evalId}.json`), JSON.stringify(artifact));+}++const build = (legsDir) => buildWithExclusions(legsDir).records;+const buildWithExclusions = (legsDir) =>+ buildRecords({+ plan,+ legs: readLegs(plan, legsDir),+ githubRunId: "123",+ runAttempt: 1,+ });++test("builds records and turns missing legs and evals into missing records", (t) => {+ const records = build(legsDirectory(t));+ assert.deepEqual(+ records.map((r) => [r.model_id, r.eval_id, r.missing, r.artifact_path]),+ [+ ["acme/alpha", "static-tools/ends-turn-function", null, "raw/agent-tools/alpha/local/1"],+ ["acme/alpha", "static-tools/no-reply", "eval-missing", "raw/agent-tools/alpha/local/1"],+ ["mock", "static-tools/ends-turn-function", "job-missing", null],+ ["mock", "static-tools/no-reply", "job-missing", null],+ ],+ );+ const [completed] = records;+ assert.equal(completed.eve_version, "0.71.2");+ assert.equal(completed.model_release, "2026-09-22");+ assert.equal(completed.verdict, "passed");+ assert.equal(completed.duration_ms, 1500);+ assert.deepEqual(completed.assertions, [+ { name: "replied", severity: "gate", score: 1, passed: true },+ ]);+ assert.equal(records[2].model_release, null);+ assert.equal(records[2].duration_ms, null);+});++test("excludes live-leg evals whose model steps all ran on another model", (t) => {+ const legsDir = legsDirectory(t);+ const leg = join(legsDir, "benchmark-leg-live__agent-tools__alpha__local__1");+ // Some fixtures script delegation evals with eve's mock models even on live legs.+ writeEval(leg, "static-tools/no-reply", "eve-mock/notebook-parent");+ const { records, excluded } = buildWithExclusions(legsDir);+ assert.deepEqual(excluded, [+ { leg_id: "live__agent-tools__alpha__local__1", eval_id: "static-tools/no-reply" },+ ]);+ const live = records.filter((r) => r.model_id === "acme/alpha");+ assert.deepEqual(+ live.map((r) => r.eval_id),+ ["static-tools/ends-turn-function"],+ "an excluded eval is neither a result nor missing",+ );+});++test("rejects artifacts that were not planned or ran at another commit", (t) => {+ assert.throws(+ () => readLegs(plan, legsDirectory(t, { sha: "b".repeat(40) })),+ /ran at b+, but the plan is for a+/,+ );+ const unplanned = { ...plan, legs: plan.legs.slice(1) };+ assert.throws(() => readLegs(unplanned, legsDirectory(t)), /was not planned/);+});++function memoryStore() {+ const objects = new Map();+ const writes = [];+ return {+ objects,+ writes,+ async put(pathname, body) {+ if (objects.has(pathname))+ throw new Error(`Vercel Blob: This blob already exists (${pathname}).`);+ objects.set(pathname, Buffer.from(body));+ writes.push(pathname);+ },+ async read(pathname) {+ return objects.get(pathname) ?? null;+ },+ };+}++test("publishes raw trees and records before the manifest, idempotently and immutably", async (t) => {+ const legsDir = legsDirectory(t);+ const legs = readLegs(plan, legsDir);+ const recordsJsonl = build(legsDir)+ .map((record) => `${JSON.stringify(record)}\n`)+ .join("");+ const store = memoryStore();+ const input = {+ plan,+ legs,+ recordsJsonl,+ githubRunId: "123",+ runAttempt: 1,+ eveVersion: "0.71.2",+ store,+ };++ const result = await publishRun(input);+ assert.equal(result.prefix, "runs/123/1");+ assert.equal(store.writes.at(-1), "runs/123/1/manifest.json");+ assert(+ store.objects.has(+ "runs/123/1/raw/agent-tools/alpha/local/1/run/evals/static-tools/ends-turn-function.json",+ ),+ );+ assert(store.objects.has("runs/123/1/raw/agent-tools/alpha/local/1/meta.json"));+ const manifest = JSON.parse(store.objects.get("runs/123/1/manifest.json"));+ assert.equal(manifest.record_count, 4);+ assert.deepEqual(+ manifest.legs.map((leg) => leg.uploaded),+ [true, false],+ );++ const writes = store.writes.length;+ await publishRun(input);+ assert.equal(store.writes.length, writes, "identical re-run is a no-op");++ await assert.rejects(+ publishRun({ ...input, recordsJsonl: recordsJsonl.split("\n").slice(1).join("\n") }),+ /records.jsonl was already published with different content/,+ );+});