Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

docs(eve): document the code extension with live DeepSWE-lean results (#4271)

Rui4b880a5ee07d
diff --git a/.changeset/eve-code-docs.md b/.changeset/eve-code-docs.md
new file mode 100644
index 000000000..a4348bb51
--- /dev/null
+++ b/.changeset/eve-code-docs.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+The packaged docs now include a Code Extension page for `eve/extensions/code`, with its latest eve-code benchmark results inlined as Markdown tables so agents reading `node_modules/eve/docs` see the numbers.
diff --git a/.github/scripts/eve-code-docs-snapshot.mjs b/.github/scripts/eve-code-docs-snapshot.mjs
new file mode 100644
index 000000000..37191f06b
--- /dev/null
+++ b/.github/scripts/eve-code-docs-snapshot.mjs
@@ -0,0 +1,133 @@
+// Turns an eve-bench report.json into the public eve-code results snapshot the docs render.
+// The snapshot keeps only aggregate scores and public run links: the result store is private,
+// so blob URLs and per-trial data never reach the docs.
+import { readFile, writeFile } from "node:fs/promises";
+
+export const SNAPSHOT_SCHEMA_VERSION = 1;
+
+/** Returns the dataset section for a report, or a reason why it must not be published. */
+export function snapshotFromReport(report) {
+ if (report.status !== "comparable" && report.status !== "measured") {
+ return { skip: `report status is ${report.status}` };
+ }
+ const manifest = report.manifest;
+ if (manifest.task !== null || manifest.cohort !== null) return { skip: "not a full-dataset run" };
+ const comparison = report.comparison?.comparison;
+ if (!comparison?.compatible || !comparison.complete || !comparison.summary) {
+ return { skip: "report has no complete cross-harness comparison" };
+ }
+ const results = new Map(comparison.results.map((result) => [result.harness, result]));
+ if (!results.has("eve-code")) return { skip: "report has no eve-code result" };
+
+ const publishedAt = new Map(
+ (report.comparison.references ?? []).map((reference) => [
+ reference.harness,
+ reference.entry.publishedAt,
+ ]),
+ );
+ const finishedAt = (result) =>
+ result.trials
+ .map((trial) => trial.finishedAt)
+ .filter(Boolean)
+ .sort()
+ .at(-1) ?? null;
+
+ return {
+ section: {
+ dataset: { name: manifest.dataset.name, version: manifest.dataset.version },
+ model: manifest.modelVersion ?? { id: manifest.model },
+ attempts: manifest.attempts,
+ tasks: manifest.tasks.length,
+ harnesses: comparison.summary.scores.map((score) => {
+ const result = results.get(score.contender);
+ if (!result) throw new Error(`No result for contender ${score.contender}.`);
+ return {
+ harness: score.contender,
+ version: harnessVersion(score.contender, result.provenance),
+ resolved: score.resolved,
+ attempts: score.attempts,
+ resolveRate: score.resolveRate,
+ costUsd: score.costUsd ?? null,
+ costPerResolvedUsd: score.costPerResolvedUsd ?? null,
+ inputTokens: score.inputTokens,
+ outputTokens: score.outputTokens,
+ cachedShare: score.cachedShare ?? null,
+ latencyP50Ms: score.latencyP50Ms,
+ latencyP90Ms: score.latencyP90Ms,
+ source: result.provenance.source?.revision ?? null,
+ runUrl: result.provenance.run?.url ?? null,
+ measuredAt: publishedAt.get(score.contender) ?? finishedAt(result),
+ };
+ }),
+ },
+ };
+}
+
+/** A released harness reports its package version; eve-code is identified by its source commit. */
+function harnessVersion(harness, provenance) {
+ const settings = provenance.harness ?? {};
+ const version = settings.version ?? settings[`${harness}Version`];
+ if (typeof version === "string") return version;
+ const sha = settings.source?.gitSha ?? provenance.source?.revision;
+ return typeof sha === "string" ? sha.slice(0, 7) : null;
+}
+
+/** Replaces one dataset's section in an existing snapshot. */
+export function mergeSnapshot(existing, section, generatedAt) {
+ const datasets = { ...existing?.datasets, [section.dataset.name]: { ...section, generatedAt } };
+ return { schemaVersion: SNAPSHOT_SCHEMA_VERSION, datasets };
+}
+
+/** GraphQL body for a GitHub-signed commit of the snapshot onto `expectedHeadOid`. */
+export function commitPayload({ repository, branch, expectedHeadOid, path, contents, runUrl }) {
+ return {
+ query:
+ "mutation($input: CreateCommitOnBranchInput!) { createCommitOnBranch(input: $input) { commit { url } } }",
+ variables: {
+ input: {
+ branch: { repositoryNameWithOwner: repository, branchName: branch },
+ expectedHeadOid,
+ message: {
+ headline: "docs: refresh eve-code benchmark results",
+ body: `From ${runUrl}\n\nSigned-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>`,
+ },
+ fileChanges: { additions: [{ path, contents: Buffer.from(contents).toString("base64") }] },
+ },
+ },
+ };
+}
+
+async function main(args) {
+ if (args[0] === "--commit-payload") {
+ const path = args[1];
+ const env = process.env;
+ const payload = commitPayload({
+ repository: env.GITHUB_REPOSITORY,
+ branch: "main",
+ expectedHeadOid: env.HEAD_OID,
+ path,
+ contents: await readFile(path),
+ runUrl: `${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}`,
+ });
+ process.stdout.write(JSON.stringify(payload));
+ return;
+ }
+ const [reportPath, snapshotPath] = args;
+ if (!reportPath || !snapshotPath) {
+ throw new Error("Usage: eve-code-docs-snapshot.mjs <report.json> <snapshot.json>");
+ }
+ const report = JSON.parse(await readFile(reportPath, "utf8"));
+ const { skip, section } = snapshotFromReport(report);
+ if (skip) {
+ console.log(`::notice::eve-code docs snapshot not updated: ${skip}.`);
+ return;
+ }
+ const existing = JSON.parse(await readFile(snapshotPath, "utf8"));
+ const next = mergeSnapshot(existing, section, new Date().toISOString());
+ await writeFile(snapshotPath, `${JSON.stringify(next, null, 2)}\n`);
+ console.log(`Updated ${section.dataset.name} in ${snapshotPath}.`);
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+ await main(process.argv.slice(2));
+}
diff --git a/.github/scripts/eve-code-docs-snapshot.test.mjs b/.github/scripts/eve-code-docs-snapshot.test.mjs
new file mode 100644
index 000000000..ead754dd2
--- /dev/null
+++ b/.github/scripts/eve-code-docs-snapshot.test.mjs
@@ -0,0 +1,89 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { mergeSnapshot, snapshotFromReport } from "./eve-code-docs-snapshot.mjs";
+
+const score = (contender, resolved) => ({
+ contender,
+ attempts: 10,
+ resolved,
+ resolveRate: { estimate: resolved / 10, low: 0, high: 1 },
+ costUsd: 1,
+ costPerResolvedUsd: 1 / resolved,
+ inputTokens: 100,
+ cachedShare: 0.5,
+ outputTokens: 10,
+ latencyP50Ms: 1000,
+ latencyP90Ms: 2000,
+});
+
+const result = (harness) => ({
+ harness,
+ trials: [{ finishedAt: "2026-09-29T00:00:00.000Z" }],
+ provenance: {
+ harness:
+ harness === "eve-code"
+ ? { source: { gitSha: "a7b0fb2b5c" } }
+ : { opencodeVersion: "1.18.30" },
+ source: { revision: "abc" },
+ run: { url: `https://example.test/${harness}` },
+ },
+});
+
+const report = (overrides = {}) => ({
+ status: "comparable",
+ manifest: {
+ task: null,
+ cohort: null,
+ dataset: { name: "swe-lean", version: "v1" },
+ model: "anthropic/claude-sonnet-5.5",
+ modelVersion: { id: "anthropic/claude-sonnet-5.5", released: "2026-09-28" },
+ attempts: 5,
+ tasks: ["a", "b"],
+ },
+ comparison: {
+ references: [{ harness: "opencode", entry: { publishedAt: "2026-09-30T00:00:00.000Z" } }],
+ comparison: {
+ compatible: true,
+ complete: true,
+ results: [result("eve-code"), result("opencode")],
+ summary: { scores: [score("eve-code", 8), score("opencode", 7)] },
+ },
+ },
+ ...overrides,
+});
+
+test("summarizes each harness with its own measurement time", () => {
+ const { section } = snapshotFromReport(report());
+ assert.equal(section.tasks, 2);
+ assert.deepEqual(
+ section.harnesses.map(({ harness, version, resolved, measuredAt }) => [
+ harness,
+ version,
+ resolved,
+ measuredAt,
+ ]),
+ [
+ ["eve-code", "a7b0fb2", 8, "2026-09-29T00:00:00.000Z"],
+ ["opencode", "1.18.30", 7, "2026-09-30T00:00:00.000Z"],
+ ],
+ );
+});
+
+test("never publishes an incomplete, partial, or eve-code-less run", () => {
+ assert.match(snapshotFromReport(report({ status: "incomplete" })).skip, /incomplete/);
+ const probe = report();
+ probe.manifest.task = "a";
+ assert.match(snapshotFromReport(probe).skip, /full-dataset/);
+ const peerless = report();
+ peerless.comparison.comparison.results = [result("opencode")];
+ assert.match(snapshotFromReport(peerless).skip, /no eve-code/);
+});
+
+test("replaces only the measured dataset", () => {
+ const existing = { schemaVersion: 1, datasets: { "deepswe-lean": { kept: true } } };
+ const { section } = snapshotFromReport(report());
+ const merged = mergeSnapshot(existing, section, "2026-10-01T00:00:00.000Z");
+ assert.deepEqual(Object.keys(merged.datasets).sort(), ["deepswe-lean", "swe-lean"]);
+ assert.equal(merged.datasets["swe-lean"].generatedAt, "2026-10-01T00:00:00.000Z");
+});
diff --git a/.github/workflows/eve-code-benchmark.yml b/.github/workflows/eve-code-benchmark.yml
index d844cc497..1b9684cdf 100644
--- a/.github/workflows/eve-code-benchmark.yml
+++ b/.github/workflows/eve-code-benchmark.yml
@@ -123,3 +123,47 @@ jobs:
vercel-token: ${{ secrets.VERCEL_TOKEN }}
vercel-team-id: ${{ secrets.VERCEL_ORG_ID }}
vercel-project-id: ${{ secrets.VERCEL_PROJECT_ID }}
+
+ docs-snapshot:
+ name: Refresh docs benchmark results
+ needs: [request, benchmark]
+ # The docs report deepswe-lean only, which runs from manual dispatches on main;
+ # only trusted main runs publish to the result store.
+ if: >-
+ github.ref == 'refs/heads/main' &&
+ github.event_name == 'workflow_dispatch' &&
+ needs.request.outputs.dataset == 'deepswe-lean'
+ concurrency:
+ group: eve-code-docs-snapshot
+ cancel-in-progress: false
+ runs-on: ubuntu-latest
+ permissions:
+ contents: write
+ actions: read
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+ - name: Download benchmark report
+ env:
+ GH_TOKEN: ${{ github.token }}
+ ARTIFACT: ${{ needs.request.outputs.dataset }}-${{ needs.request.outputs.slug }}
+ run: gh run download "$GITHUB_RUN_ID" -R "$GITHUB_REPOSITORY" -n "$ARTIFACT" -D "$RUNNER_TEMP/bench"
+ - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+ with:
+ node-version: 24
+ # Merge onto the newest main so a concurrent refresh of the other dataset
+ # is never overwritten. createCommitOnBranch commits are signed by GitHub,
+ # which satisfies the signed-commit requirement without a bot signing key.
+ - name: Commit snapshot to main
+ env:
+ GH_TOKEN: ${{ github.token }}
+ FILE: apps/docs/lib/evals/eve-code-benchmark.json
+ run: |
+ head=$(gh api "repos/$GITHUB_REPOSITORY/git/ref/heads/main" -q .object.sha)
+ gh api "repos/$GITHUB_REPOSITORY/contents/$FILE?ref=$head" -H "Accept: application/vnd.github.raw" > "$FILE"
+ cp "$FILE" "$RUNNER_TEMP/before.json"
+ node .github/scripts/eve-code-docs-snapshot.mjs "$RUNNER_TEMP/bench/report.json" "$FILE"
+ if cmp -s "$FILE" "$RUNNER_TEMP/before.json"; then exit 0; fi
+ HEAD_OID="$head" node .github/scripts/eve-code-docs-snapshot.mjs --commit-payload "$FILE" > "$RUNNER_TEMP/commit.json"
+ gh api graphql --input "$RUNNER_TEMP/commit.json" -q .data.createCommitOnBranch.commit.url
diff --git a/apps/docs/components/geistdocs/eve-code-benchmark.tsx b/apps/docs/components/geistdocs/eve-code-benchmark.tsx
new file mode 100644
index 000000000..12f6c6eec
--- /dev/null
+++ b/apps/docs/components/geistdocs/eve-code-benchmark.tsx
@@ -0,0 +1,213 @@
+"use client";
+
+import { useState } from "react";
+
+import {
+ eveCodeBenchmark,
+ rankedHarnesses,
+ type EveCodeHarnessScore,
+} from "@/lib/evals/eve-code-results";
+
+const REFERENCE = "eve-code";
+const COLORS = ["#4f6bed", "#5ed37f", "#e0803f", "#c26bd9", "#d9c24f"];
+
+interface Metric {
+ label: string;
+ higherIsBetter: boolean;
+ value: (score: EveCodeHarnessScore) => number;
+ /** Interval drawn around the dot, when the metric has one. */
+ range?: (score: EveCodeHarnessScore) => [number, number];
+ format: (value: number) => string;
+ /** Difference from the reference contender. */
+ delta: (value: number, reference: number) => string;
+ /** Fixed axis maximum; otherwise fit to the data. */
+ max?: number;
+}
+
+const signed = (value: number, unit: string) =>
+ `${value > 0 ? "+" : value < 0 ? "−" : "±"}${Math.abs(Math.round(value))}${unit}`;
+const relative = (value: number, reference: number) =>
+ reference === 0 ? "—" : signed((value / reference - 1) * 100, "%");
+
+const METRICS: Record<string, Metric> = {
+ correctness: {
+ label: "correctness",
+ higherIsBetter: true,
+ value: (score) => score.resolveRate.estimate,
+ range: (score) => [score.resolveRate.low, score.resolveRate.high],
+ format: (value) => `${Math.round(value * 100)}%`,
+ delta: (value, reference) => signed((value - reference) * 100, " pp"),
+ max: 1,
+ },
+ "input tokens": {
+ label: "input tokens",
+ higherIsBetter: false,
+ value: (score) => score.inputTokens / score.attempts,
+ format: (value) =>
+ value >= 1_000_000 ? `${(value / 1_000_000).toFixed(1)}M` : `${Math.round(value / 1000)}k`,
+ delta: relative,
+ },
+ latency: {
+ label: "latency",
+ higherIsBetter: false,
+ value: (score) => score.latencyP50Ms,
+ range: (score) => [score.latencyP50Ms, score.latencyP90Ms],
+ format: (value) => `${Math.round(value / 1000)}s`,
+ delta: relative,
+ },
+};
+
+/** Renders the latest published eve-code results for one eve-bench dataset. */
+export const EveCodeBenchmark = ({ dataset }: { dataset: string }) => {
+ const [metricKey, setMetricKey] = useState("correctness");
+ const results = eveCodeBenchmark.datasets[dataset];
+ if (!results) return <p>No published {dataset} results yet.</p>;
+
+ const metric = METRICS[metricKey];
+ const rows = rankedHarnesses(results);
+ const colors = new Map(
+ rows.map((score, index) => [score.harness, COLORS[index % COLORS.length]]),
+ );
+ const reference = rows.find((score) => score.harness === REFERENCE);
+ const max =
+ metric.max ??
+ niceCeiling(Math.max(...rows.map((score) => metric.range?.(score)[1] ?? metric.value(score))));
+ const ticks = [0, 0.25, 0.5, 0.75, 1].map((fraction) => fraction * max);
+ const best = (metric.higherIsBetter ? Math.max : Math.min)(...rows.map(metric.value));
+ const position = (value: number) => `${(value / max) * 100}%`;
+ const runs = new Set(rows.map((score) => score.runUrl)).size;
+ const sorted = [...rows].sort((left, right) =>
+ metric.higherIsBetter
+ ? metric.value(right) - metric.value(left)
+ : metric.value(left) - metric.value(right),
+ );
+
+ return (
+ <figure className="not-prose my-6 overflow-x-auto rounded-xl border border-gray-400 bg-background-100 p-4 text-sm">
+ <figcaption className="flex flex-wrap items-baseline gap-x-3 gap-y-1">
+ <span className="font-mono font-medium text-gray-1000">
+ {results.dataset.name}@{results.dataset.version} · {results.model.id.split("/").at(-1)}
+ </span>
+ <span className="text-gray-800">
+ {rows.length} contenders across {runs} run{runs === 1 ? "" : "s"}
+ </span>
+ </figcaption>
+
+ <div
+ aria-label="Metric"
+ className="mt-3 inline-flex rounded-lg border border-gray-400 p-0.5"
+ role="tablist"
+ >
+ {Object.keys(METRICS).map((key) => (
+ <button
+ aria-selected={key === metricKey}
+ className={`rounded-md px-3 py-1 ${key === metricKey ? "bg-gray-200 text-gray-1000" : "text-gray-800 hover:text-gray-1000"}`}
+ key={key}
+ onClick={() => setMetricKey(key)}
+ role="tab"
+ type="button"
+ >
+ {key}
+ </button>
+ ))}
+ </div>
+
+ <div className="mt-4 min-w-[640px]" role="tabpanel">
+ <div className="grid grid-cols-[12rem_1fr_6rem_5rem] items-end gap-x-4 pb-1 text-gray-800">
+ <span>{metric.higherIsBetter ? "higher is better" : "lower is better"}</span>
+ <div className="relative h-5 font-mono text-xs">
+ {ticks.map((tick) => (
+ <span
+ className="absolute -translate-x-1/2"
+ key={tick}
+ style={{ left: position(tick) }}
+ >
+ {metric.format(tick)}
+ </span>
+ ))}
+ </div>
+ <span className="text-right">{metric.label}</span>
+ <span className="text-right">vs ref</span>
+ </div>
+
+ {sorted.map((score) => {
+ const value = metric.value(score);
+ const color = colors.get(score.harness);
+ const range = metric.range?.(score);
+ const isReference = score.harness === REFERENCE;
+ return (
+ <div
+ className="grid grid-cols-[12rem_1fr_6rem_5rem] items-center gap-x-4 rounded-md py-1.5 hover:bg-gray-100"
+ key={score.harness}
+ >
+ <span className="flex min-w-0 items-center gap-2 font-mono text-gray-1000">
+ <span className="size-2.5 shrink-0 rounded-full" style={{ background: color }} />
+ <span className="truncate">
+ {score.runUrl ? (
+ <a href={score.runUrl} title="Open the GitHub Actions run">
+ {score.harness}
+ {score.version ? `@${score.version}` : ""}
+ </a>
+ ) : (
+ `${score.harness}${score.version ? `@${score.version}` : ""}`
+ )}
+ </span>
+ </span>
+
+ <div className="relative h-5">
+ {ticks.map((tick) => (
+ <span
+ className="absolute inset-y-0 w-px bg-gray-300"
+ key={tick}
+ style={{ left: position(tick) }}
+ />
+ ))}
+ {range ? (
+ <span
+ className="absolute top-1/2 h-0.5 -translate-y-1/2 opacity-60"
+ style={{
+ background: color,
+ left: position(range[0]),
+ width: `calc(${position(range[1])} - ${position(range[0])})`,
+ }}
+ />
+ ) : null}
+ <span
+ aria-label={`${score.harness}: ${metric.format(value)}${range ? `, interval ${metric.format(range[0])} to ${metric.format(range[1])}` : ""}`}
+ className="absolute top-1/2 size-3.5 -translate-x-1/2 -translate-y-1/2 rounded-full ring-2 ring-background-100 transition-[left] duration-300"
+ role="img"
+ style={{ background: color, left: position(value) }}
+ />
+ </div>
+
+ <span
+ className={`text-right font-mono ${value === best ? "text-green-700" : "text-gray-1000"}`}
+ >
+ {metric.format(value)}
+ </span>
+ <span className="text-right font-mono text-gray-900">
+ {isReference || !reference ? "ref" : metric.delta(value, metric.value(reference))}
+ </span>
+ </div>
+ );
+ })}
+ </div>
+
+ <p className="mt-3 text-gray-700 text-xs">
+ {metric.range
+ ? metricKey === "latency"
+ ? "Dots are median latency; lines extend to p90."
+ : "Lines are 95% intervals; overlapping lines are not a measured difference."
+ : "Mean input tokens per attempt."}{" "}
+ Click a contender to open its run.
+ </p>
+ </figure>
+ );
+};
+
+function niceCeiling(value: number): number {
+ if (value <= 0) return 1;
+ const magnitude = 10 ** Math.floor(Math.log10(value));
+ const step = [1, 2, 2.5, 5, 10].find((candidate) => candidate * magnitude >= value) ?? 10;
+ return step * magnitude;
+}
diff --git a/apps/docs/components/geistdocs/mdx-components.tsx b/apps/docs/components/geistdocs/mdx-components.tsx
index 1f5152b2f..5ed5bcbfc 100644
--- a/apps/docs/components/geistdocs/mdx-components.tsx
+++ b/apps/docs/components/geistdocs/mdx-components.tsx
@@ -5,6 +5,7 @@ import { Flag } from "lucide-react";
import type { MDXComponents } from "mdx/types";
import Link from "next/link";
import { AgentRuntimeDiagram } from "./agent-runtime-diagram";
+import { EveCodeBenchmark } from "./eve-code-benchmark";
const localComponents: MDXComponents = {
a: ({ href, ...props }) =>
@@ -23,6 +24,7 @@ const localComponents: MDXComponents = {
Flag,
Folder,
AgentRuntimeDiagram,
+ EveCodeBenchmark,
Step,
Steps,
};
diff --git a/apps/docs/lib/evals/eve-code-benchmark.json b/apps/docs/lib/evals/eve-code-benchmark.json
new file mode 100644
index 000000000..b65b62b32
--- /dev/null
+++ b/apps/docs/lib/evals/eve-code-benchmark.json
@@ -0,0 +1,84 @@
+{
+ "schemaVersion": 1,
+ "datasets": {
+ "deepswe-lean": {
+ "dataset": {
+ "name": "deepswe-lean",
+ "version": "v1"
+ },
+ "model": {
+ "id": "anthropic/claude-sonnet-5.5",
+ "released": "2026-09-28",
+ "knowledge": "2026-06"
+ },
+ "attempts": 3,
+ "tasks": 12,
+ "harnesses": [
+ {
+ "harness": "eve-code",
+ "version": "4d14914",
+ "resolved": 24,
+ "attempts": 36,
+ "resolveRate": {
+ "estimate": 0.6666666666666666,
+ "low": 0.5277777777777776,
+ "high": 0.8055555555555557
+ },
+ "costUsd": 20.336862000000004,
+ "costPerResolvedUsd": 0.8473692500000002,
+ "inputTokens": 35312195,
+ "outputTokens": 865005,
+ "cachedShare": 0.94331136877784,
+ "latencyP50Ms": 235438.5,
+ "latencyP90Ms": 389674,
+ "source": "4d1491477a2e4f220131a3eb857a030c9eec0419",
+ "runUrl": "https://github.com/vercel/eve/actions/runs/36775772040",
+ "measuredAt": "2026-09-30T21:11:47.181Z"
+ },
+ {
+ "harness": "opencode",
+ "version": "1.18.30",
+ "resolved": 18,
+ "attempts": 36,
+ "resolveRate": {
+ "estimate": 0.5,
+ "low": 0.27777777777777773,
+ "high": 0.7222222222222223
+ },
+ "costUsd": null,
+ "costPerResolvedUsd": null,
+ "inputTokens": 25457380,
+ "outputTokens": 2837675,
+ "cachedShare": 0.9378020047624697,
+ "latencyP50Ms": 611686.5,
+ "latencyP90Ms": 1170330.5,
+ "source": "a7b0fb2bf24d57e6bb3613d557c0af53a171d558",
+ "runUrl": "https://github.com/vercel/eve/actions/runs/36661642270",
+ "measuredAt": "2026-09-30T03:33:15.672Z"
+ },
+ {
+ "harness": "pi",
+ "version": "0.85.1",
+ "resolved": 26,
+ "attempts": 36,
+ "resolveRate": {
+ "estimate": 0.7222222222222222,
+ "low": 0.5,
+ "high": 0.9166666666666666
+ },
+ "costUsd": null,
+ "costPerResolvedUsd": null,
+ "inputTokens": 27007661,
+ "outputTokens": 903678,
+ "cachedShare": 0,
+ "latencyP50Ms": 266711,
+ "latencyP90Ms": 340482.5,
+ "source": "a7b0fb2bf24d57e6bb3613d557c0af53a171d558",
+ "runUrl": "https://github.com/vercel/eve/actions/runs/36661642270",
+ "measuredAt": "2026-09-30T03:33:15.823Z"
+ }
+ ],
+ "generatedAt": "2026-10-03T03:29:12.252Z"
+ }
+ }
+}
diff --git a/apps/docs/lib/evals/eve-code-results.ts b/apps/docs/lib/evals/eve-code-results.ts
new file mode 100644
index 000000000..977d68a38
--- /dev/null
+++ b/apps/docs/lib/evals/eve-code-results.ts
@@ -0,0 +1,46 @@
+import rawSnapshot from "./eve-code-benchmark.json";
+
+/** Written by `.github/scripts/eve-code-docs-snapshot.mjs` from eve-bench reports of `main`. */
+export interface EveCodeHarnessScore {
+ harness: string;
+ /** Package version for released harnesses; short source commit for eve-code. */
+ version: string | null;
+ resolved: number;
+ attempts: number;
+ resolveRate: { estimate: number; low: number; high: number };
+ costUsd: number | null;
+ costPerResolvedUsd: number | null;
+ inputTokens: number;
+ outputTokens: number;
+ cachedShare: number | null;
+ latencyP50Ms: number;
+ latencyP90Ms: number;
+ source: string | null;
+ runUrl: string | null;
+ measuredAt: string | null;
+}
+
+export interface EveCodeDatasetResults {
+ dataset: { name: string; version: string };
+ model: { id: string; released?: string; knowledge?: string };
+ attempts: number;
+ tasks: number;
+ generatedAt: string;
+ harnesses: EveCodeHarnessScore[];
+}
+
+export interface EveCodeBenchmarkSnapshot {
+ schemaVersion: 1;
+ datasets: Record<string, EveCodeDatasetResults>;
+}
+
+export const eveCodeBenchmark = rawSnapshot as EveCodeBenchmarkSnapshot;
+
+/** Ranks by resolve rate, then by median latency, matching eve-bench's correctness-first order. */
+export function rankedHarnesses(results: EveCodeDatasetResults): EveCodeHarnessScore[] {
+ return [...results.harnesses].sort(
+ (left, right) =>
+ right.resolveRate.estimate - left.resolveRate.estimate ||
+ left.latencyP50Ms - right.latencyP50Ms,
+ );
+}
diff --git a/docs/README.md b/docs/README.md
index c0c065419..b10ae11d6 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -28,6 +28,7 @@ Important naming note:
| Delegate work to a specialist child agent | [Subagents](./subagents/index.mdx) |
| Run work on a recurring schedule | [Schedules](./schedules.mdx) |
| Install an existing integration instead of writing one | [Add Integrations](./install-integrations.mdx) |
+| Build a coding agent, or compare eve-code benchmarks | [Code Extension](./code-extension.mdx) |
| Link a Vercel project and deploy to production | [Deploy to Vercel](./guides/deployment/vercel.mdx) |
| Self-host, or compare hosting strategies | [Deployment](./guides/deployment/overview.md) |
| Authorize routes, sessions, and per-user access | [Authentication](./guides/auth-and-route-protection.md) |
diff --git a/docs/code-extension.mdx b/docs/code-extension.mdx
new file mode 100644
index 000000000..b5a8e0382
--- /dev/null
+++ b/docs/code-extension.mdx
@@ -0,0 +1,113 @@
+---
+title: "Code Extension"
+description: "Mount eve's coding extension, eve-code, and see its latest benchmark results against other coding agent harnesses."
+---
+
+`eve/extensions/code` (eve-code) turns an eve agent into a coding agent. It ships inside the `eve` package, so you mount it without installing anything else. The extension contributes file-editing and search tools, an authenticated GitHub CLI tool, coding skills, a read-only worker subagent, and sandbox bootstrap helpers.
+
+## Benchmark results
+
+<EveCodeBenchmark dataset="deepswe-lean" />
+
+These results show baseline performance. Each harness uses its standard setup with high reasoning. The results do not show how a harness performs with other models, settings, or tasks.
+
+<details>
+<summary>How the benchmark works</summary>
+
+The benchmark uses eve-bench and the DeepSWE-lean dataset. DeepSWE-lean has 12 feature and bug-fix tasks from DeepSWE v1.1.
+
+- All harnesses use the same model and the same tasks.
+- Each harness does each task 3 times.
+- Each attempt runs in a new Vercel Sandbox.
+- Each attempt has a limit of 20 minutes.
+- The DeepSWE verifier examines the files that the agent leaves.
+- An attempt passes only when the verifier passes.
+- A crash or a timeout is a failure.
+
+The line on each correctness row shows the 95% interval. When the lines of two harnesses overlap, the benchmark does not show a difference between them.
+
+</details>
+
+<details>
+<summary>How the results update</summary>
+
+1. A maintainer runs the `eve-code` workflow on `main` with the `deepswe-lean` dataset.
+2. The workflow runs eve-code, opencode, and pi. It saves each complete result in the eve-bench result store.
+3. The workflow writes a summary of the result to `apps/docs/lib/evals/eve-code-benchmark.json`. Then it commits the file to `main`.
+4. The next docs deployment shows the new results.
+
+The workflow does not publish incomplete runs, single-task runs, or pull request runs. Results can be older than `main`, because a maintainer starts each run. Select a harness name to open the GitHub Actions run for that result.
+
+</details>
+
+## Mount the extension
+
+Create an extension file in your agent. The file name sets the mount name, which prefixes every contributed tool and subagent ID (for example, `code__grep`):
+
+```ts title="agent/extensions/code.ts"
+import code from "eve/extensions/code";
+
+export default code({});
+```
+
+`code({})` mounts the extension without GitHub or Vercel authentication. Add the options below when the agent needs them.
+
+## Configure the extension
+
+| Option | Required | Effect |
+| ------------------------------ | ------------- | ---------------------------------------------------------------------------------------------- |
+| `github.connector` | With `github` | Connector that mints a GitHub token |
+| `github.org` | With `github` | Organization whose repositories the `gh` tool can access |
+| `github.broker` | With `github` | `broker(sandbox, rules)` callback that installs, then removes, the GitHub credential rules |
+| `vercel.connector` | With `vercel` | Connector that requests an app-subject Vercel token |
+| `vercel.delivery` | No | `"firewall"` (default) or `"command"` |
+| `broker` | No | `broker(sandbox, rules)` callback that merges Vercel credential rules into your network policy |
+| `worker.model` | With `worker` | Model for the `worker` subagent. Defaults to `openai/gpt-5.6-terra-fast` |
+| `worker.reasoning` | With `worker` | Reasoning level for the `worker` subagent. Defaults to `xhigh` |
+| `worker.openaiReasoningEffort` | No | Passed to OpenAI models as `reasoningEffort` |
+
+```ts title="agent/extensions/code.ts"
+import code from "eve/extensions/code";
+
+export default code({
+ vercel: { connector: "vercel/acme-bot" },
+ worker: { model: "openai/gpt-5.6-terra-fast", reasoning: "high" },
+});
+```
+
+### Credentials stay outside the sandbox
+
+With firewall delivery, Vercel tokens never enter sandbox processes. Firewall delivery without a top-level `broker` requires a sandbox provider that exposes `setNetworkPolicy()`, and fails otherwise. Set `vercel.delivery: "command"` for providers without a mutable network policy.
+
+The `gh` tool requests a token scoped to exactly one repository in `github.org` for each invocation. The sandbox process receives a placeholder `GH_TOKEN`, and the firewall exchanges it on matching GitHub requests. Your `github.broker` callback receives the header-transform rules for the command, then `null` to remove the lease. The callback must preserve your other network rules. GitHub authentication has no command-delivery option.
+
+To use a personal access token or another credential provider, omit the connector and call `authenticateGitHub` or `authenticateVercel` from `eve/extensions/code/sandbox` in your own sandbox lifecycle. These helpers authenticate commands you run yourself; they do not configure the `gh` tool.
+
+## Prepare the sandbox
+
+Install the CLI tooling in your sandbox environment's `prepare` callback:
+
+```ts title="agent/sandbox.ts"
+import { defineSandbox } from "eve/sandbox";
+import { VercelSandbox } from "eve/sandbox/vercel";
+import { installCodeTooling } from "eve/extensions/code/sandbox";
+
+export const environment = VercelSandbox.environment({
+ prepare: async (sandbox) => {
+ await installCodeTooling(sandbox, { vercel: true });
+ },
+});
+
+export default defineSandbox(() => environment.open());
+```
+
+Preparation installs `gh`, wrappers for `gh`, `vc`, and `gh-signed-commit`, and TypeScript diagnostics. For repositories that require verified signatures, stage your changes and commit with `gh-signed-commit`.
+
+eve derives the prepared environment generation from the sandbox file and environment options, not from imported helpers. Upgrading eve alone does not rebuild an existing prepared environment.
+
+## What to read next
+
+- [Extensions](/docs/extensions): mount, configure, and override extension packages
+- [Sandbox](/docs/sandbox): environments, preparation, and network policy
+- [Subagents](/docs/subagents): how declared subagents run and inherit configuration
+- [Skills](/docs/skills): how the model loads skills on demand
diff --git a/docs/extensions.md b/docs/extensions.md
index bc5dcf447..a039bd1ac 100644
--- a/docs/extensions.md
+++ b/docs/extensions.md
@@ -370,6 +370,7 @@ At build time, eve checks the extension's generated capability metadata. If the
## What to read next
- [Integrations](/integrations): browse ready-to-install extensions using the Extensions filter
+- [Code extension](/docs/code-extension): mount eve-code, the coding extension that ships in `eve`, and see its benchmark results
- [Tools](/docs/tools): static tools, approval, and tool output
- [Dynamic capabilities](/docs/guides/dynamic-capabilities): dynamic connections, tools, skills, and instructions
- [Instructions](/docs/instructions): static and TypeScript instructions
diff --git a/docs/meta.json b/docs/meta.json
index 0dcbf7570..44e421fa0 100644
--- a/docs/meta.json
+++ b/docs/meta.json
@@ -20,6 +20,7 @@
"---Integrate---",
"install-integrations",
"extensions",
+ "code-extension",
"protocols/acp",
"guides/client",
"guides/frontend",
diff --git a/packages/eve/scripts/copy-docs.mjs b/packages/eve/scripts/copy-docs.mjs
index 27b449435..51873bb69 100644
--- a/packages/eve/scripts/copy-docs.mjs
+++ b/packages/eve/scripts/copy-docs.mjs
@@ -1,4 +1,4 @@
-import { cp, rm } from "node:fs/promises";
+import { cp, readFile, rm, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
@@ -14,3 +14,42 @@ const oldDistDocsDir = join(packageRoot, "dist", "docs");
await rm(packageDocsDir, { recursive: true, force: true });
await rm(oldDistDocsDir, { recursive: true, force: true });
await cp(join(monorepoRoot, "docs"), packageDocsDir, { recursive: true });
+
+// Agents read the raw file, where the site's <EveCodeBenchmark /> component
+// would render nothing, so inline the same snapshot as a Markdown table.
+const benchmarkPage = join(packageDocsDir, "code-extension.mdx");
+const page = await readFile(benchmarkPage, "utf8").catch((error) => {
+ if (error.code === "ENOENT") return null;
+ throw error;
+});
+if (page !== null) {
+ const snapshot = JSON.parse(
+ await readFile(join(monorepoRoot, "apps/docs/lib/evals/eve-code-benchmark.json"), "utf8"),
+ );
+ await writeFile(
+ benchmarkPage,
+ page.replace(/<EveCodeBenchmark dataset="([^"]+)" \/>/gu, (_, dataset) =>
+ benchmarkTable(snapshot.datasets[dataset], dataset),
+ ),
+ );
+}
+
+function benchmarkTable(results, dataset) {
+ if (!results) return `No published ${dataset} results yet.`;
+ const pct = (value) => `${Math.round(value * 100)}%`;
+ const rows = [...results.harnesses]
+ .sort(
+ (a, b) => b.resolveRate.estimate - a.resolveRate.estimate || a.latencyP50Ms - b.latencyP50Ms,
+ )
+ .map(
+ (h) =>
+ `| \`${h.harness}\` | ${pct(h.resolveRate.estimate)} (${h.resolved}/${h.attempts}) | ${pct(h.resolveRate.low)}–${pct(h.resolveRate.high)} | ${(h.latencyP50Ms / 1000).toFixed(1)}s | ${(h.latencyP90Ms / 1000).toFixed(1)}s | ${h.measuredAt?.slice(0, 10) ?? "—"} |`,
+ );
+ return [
+ `${results.tasks} tasks × ${results.attempts} attempts on \`${results.model.id}\`. Snapshot updated ${results.generatedAt.slice(0, 10)}.`,
+ "",
+ "| Harness | Resolved | 95% interval | Median latency | p90 latency | Measured |",
+ "| --- | --- | --- | --- | --- | --- |",
+ ...rows,
+ ].join("\n");
+}