The Open Framework for Building Agents
docs(eve): document the code extension with live DeepSWE-lean results (#4271)
Rui4b880a5ee07d
diff --git a/.changeset/eve-code-docs.md b/.changeset/eve-code-docs.mdnew file mode 100644index 000000000..a4348bb51--- /dev/null+++ b/.changeset/eve-code-docs.md@@ -0,0 +1,5 @@+---+"eve": patch+---++The packaged docs now include a Code Extension page for `eve/extensions/code`, with its latest eve-code benchmark results inlined as Markdown tables so agents reading `node_modules/eve/docs` see the numbers.diff --git a/.github/scripts/eve-code-docs-snapshot.mjs b/.github/scripts/eve-code-docs-snapshot.mjsnew file mode 100644index 000000000..37191f06b--- /dev/null+++ b/.github/scripts/eve-code-docs-snapshot.mjs@@ -0,0 +1,133 @@+// Turns an eve-bench report.json into the public eve-code results snapshot the docs render.+// The snapshot keeps only aggregate scores and public run links: the result store is private,+// so blob URLs and per-trial data never reach the docs.+import { readFile, writeFile } from "node:fs/promises";++export const SNAPSHOT_SCHEMA_VERSION = 1;++/** Returns the dataset section for a report, or a reason why it must not be published. */+export function snapshotFromReport(report) {+ if (report.status !== "comparable" && report.status !== "measured") {+ return { skip: `report status is ${report.status}` };+ }+ const manifest = report.manifest;+ if (manifest.task !== null || manifest.cohort !== null) return { skip: "not a full-dataset run" };+ const comparison = report.comparison?.comparison;+ if (!comparison?.compatible || !comparison.complete || !comparison.summary) {+ return { skip: "report has no complete cross-harness comparison" };+ }+ const results = new Map(comparison.results.map((result) => [result.harness, result]));+ if (!results.has("eve-code")) return { skip: "report has no eve-code result" };++ const publishedAt = new Map(+ (report.comparison.references ?? []).map((reference) => [+ reference.harness,+ reference.entry.publishedAt,+ ]),+ );+ const finishedAt = (result) =>+ result.trials+ .map((trial) => trial.finishedAt)+ .filter(Boolean)+ .sort()+ .at(-1) ?? null;++ return {+ section: {+ dataset: { name: manifest.dataset.name, version: manifest.dataset.version },+ model: manifest.modelVersion ?? { id: manifest.model },+ attempts: manifest.attempts,+ tasks: manifest.tasks.length,+ harnesses: comparison.summary.scores.map((score) => {+ const result = results.get(score.contender);+ if (!result) throw new Error(`No result for contender ${score.contender}.`);+ return {+ harness: score.contender,+ version: harnessVersion(score.contender, result.provenance),+ resolved: score.resolved,+ attempts: score.attempts,+ resolveRate: score.resolveRate,+ costUsd: score.costUsd ?? null,+ costPerResolvedUsd: score.costPerResolvedUsd ?? null,+ inputTokens: score.inputTokens,+ outputTokens: score.outputTokens,+ cachedShare: score.cachedShare ?? null,+ latencyP50Ms: score.latencyP50Ms,+ latencyP90Ms: score.latencyP90Ms,+ source: result.provenance.source?.revision ?? null,+ runUrl: result.provenance.run?.url ?? null,+ measuredAt: publishedAt.get(score.contender) ?? finishedAt(result),+ };+ }),+ },+ };+}++/** A released harness reports its package version; eve-code is identified by its source commit. */+function harnessVersion(harness, provenance) {+ const settings = provenance.harness ?? {};+ const version = settings.version ?? settings[`${harness}Version`];+ if (typeof version === "string") return version;+ const sha = settings.source?.gitSha ?? provenance.source?.revision;+ return typeof sha === "string" ? sha.slice(0, 7) : null;+}++/** Replaces one dataset's section in an existing snapshot. */+export function mergeSnapshot(existing, section, generatedAt) {+ const datasets = { ...existing?.datasets, [section.dataset.name]: { ...section, generatedAt } };+ return { schemaVersion: SNAPSHOT_SCHEMA_VERSION, datasets };+}++/** GraphQL body for a GitHub-signed commit of the snapshot onto `expectedHeadOid`. */+export function commitPayload({ repository, branch, expectedHeadOid, path, contents, runUrl }) {+ return {+ query:+ "mutation($input: CreateCommitOnBranchInput!) { createCommitOnBranch(input: $input) { commit { url } } }",+ variables: {+ input: {+ branch: { repositoryNameWithOwner: repository, branchName: branch },+ expectedHeadOid,+ message: {+ headline: "docs: refresh eve-code benchmark results",+ body: `From ${runUrl}\n\nSigned-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>`,+ },+ fileChanges: { additions: [{ path, contents: Buffer.from(contents).toString("base64") }] },+ },+ },+ };+}++async function main(args) {+ if (args[0] === "--commit-payload") {+ const path = args[1];+ const env = process.env;+ const payload = commitPayload({+ repository: env.GITHUB_REPOSITORY,+ branch: "main",+ expectedHeadOid: env.HEAD_OID,+ path,+ contents: await readFile(path),+ runUrl: `${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}`,+ });+ process.stdout.write(JSON.stringify(payload));+ return;+ }+ const [reportPath, snapshotPath] = args;+ if (!reportPath || !snapshotPath) {+ throw new Error("Usage: eve-code-docs-snapshot.mjs <report.json> <snapshot.json>");+ }+ const report = JSON.parse(await readFile(reportPath, "utf8"));+ const { skip, section } = snapshotFromReport(report);+ if (skip) {+ console.log(`::notice::eve-code docs snapshot not updated: ${skip}.`);+ return;+ }+ const existing = JSON.parse(await readFile(snapshotPath, "utf8"));+ const next = mergeSnapshot(existing, section, new Date().toISOString());+ await writeFile(snapshotPath, `${JSON.stringify(next, null, 2)}\n`);+ console.log(`Updated ${section.dataset.name} in ${snapshotPath}.`);+}++if (import.meta.url === `file://${process.argv[1]}`) {+ await main(process.argv.slice(2));+}diff --git a/.github/scripts/eve-code-docs-snapshot.test.mjs b/.github/scripts/eve-code-docs-snapshot.test.mjsnew file mode 100644index 000000000..ead754dd2--- /dev/null+++ b/.github/scripts/eve-code-docs-snapshot.test.mjs@@ -0,0 +1,89 @@+import assert from "node:assert/strict";+import test from "node:test";++import { mergeSnapshot, snapshotFromReport } from "./eve-code-docs-snapshot.mjs";++const score = (contender, resolved) => ({+ contender,+ attempts: 10,+ resolved,+ resolveRate: { estimate: resolved / 10, low: 0, high: 1 },+ costUsd: 1,+ costPerResolvedUsd: 1 / resolved,+ inputTokens: 100,+ cachedShare: 0.5,+ outputTokens: 10,+ latencyP50Ms: 1000,+ latencyP90Ms: 2000,+});++const result = (harness) => ({+ harness,+ trials: [{ finishedAt: "2026-09-29T00:00:00.000Z" }],+ provenance: {+ harness:+ harness === "eve-code"+ ? { source: { gitSha: "a7b0fb2b5c" } }+ : { opencodeVersion: "1.18.30" },+ source: { revision: "abc" },+ run: { url: `https://example.test/${harness}` },+ },+});++const report = (overrides = {}) => ({+ status: "comparable",+ manifest: {+ task: null,+ cohort: null,+ dataset: { name: "swe-lean", version: "v1" },+ model: "anthropic/claude-sonnet-5.5",+ modelVersion: { id: "anthropic/claude-sonnet-5.5", released: "2026-09-28" },+ attempts: 5,+ tasks: ["a", "b"],+ },+ comparison: {+ references: [{ harness: "opencode", entry: { publishedAt: "2026-09-30T00:00:00.000Z" } }],+ comparison: {+ compatible: true,+ complete: true,+ results: [result("eve-code"), result("opencode")],+ summary: { scores: [score("eve-code", 8), score("opencode", 7)] },+ },+ },+ ...overrides,+});++test("summarizes each harness with its own measurement time", () => {+ const { section } = snapshotFromReport(report());+ assert.equal(section.tasks, 2);+ assert.deepEqual(+ section.harnesses.map(({ harness, version, resolved, measuredAt }) => [+ harness,+ version,+ resolved,+ measuredAt,+ ]),+ [+ ["eve-code", "a7b0fb2", 8, "2026-09-29T00:00:00.000Z"],+ ["opencode", "1.18.30", 7, "2026-09-30T00:00:00.000Z"],+ ],+ );+});++test("never publishes an incomplete, partial, or eve-code-less run", () => {+ assert.match(snapshotFromReport(report({ status: "incomplete" })).skip, /incomplete/);+ const probe = report();+ probe.manifest.task = "a";+ assert.match(snapshotFromReport(probe).skip, /full-dataset/);+ const peerless = report();+ peerless.comparison.comparison.results = [result("opencode")];+ assert.match(snapshotFromReport(peerless).skip, /no eve-code/);+});++test("replaces only the measured dataset", () => {+ const existing = { schemaVersion: 1, datasets: { "deepswe-lean": { kept: true } } };+ const { section } = snapshotFromReport(report());+ const merged = mergeSnapshot(existing, section, "2026-10-01T00:00:00.000Z");+ assert.deepEqual(Object.keys(merged.datasets).sort(), ["deepswe-lean", "swe-lean"]);+ assert.equal(merged.datasets["swe-lean"].generatedAt, "2026-10-01T00:00:00.000Z");+});diff --git a/.github/workflows/eve-code-benchmark.yml b/.github/workflows/eve-code-benchmark.ymlindex d844cc497..1b9684cdf 100644--- a/.github/workflows/eve-code-benchmark.yml+++ b/.github/workflows/eve-code-benchmark.yml@@ -123,3 +123,47 @@ jobs:vercel-token: ${{ secrets.VERCEL_TOKEN }}vercel-team-id: ${{ secrets.VERCEL_ORG_ID }}vercel-project-id: ${{ secrets.VERCEL_PROJECT_ID }}++ docs-snapshot:+ name: Refresh docs benchmark results+ needs: [request, benchmark]+ # The docs report deepswe-lean only, which runs from manual dispatches on main;+ # only trusted main runs publish to the result store.+ if: >-+ github.ref == 'refs/heads/main' &&+ github.event_name == 'workflow_dispatch' &&+ needs.request.outputs.dataset == 'deepswe-lean'+ concurrency:+ group: eve-code-docs-snapshot+ cancel-in-progress: false+ runs-on: ubuntu-latest+ permissions:+ contents: write+ actions: read+ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ persist-credentials: false+ - name: Download benchmark report+ env:+ GH_TOKEN: ${{ github.token }}+ ARTIFACT: ${{ needs.request.outputs.dataset }}-${{ needs.request.outputs.slug }}+ run: gh run download "$GITHUB_RUN_ID" -R "$GITHUB_REPOSITORY" -n "$ARTIFACT" -D "$RUNNER_TEMP/bench"+ - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0+ with:+ node-version: 24+ # Merge onto the newest main so a concurrent refresh of the other dataset+ # is never overwritten. createCommitOnBranch commits are signed by GitHub,+ # which satisfies the signed-commit requirement without a bot signing key.+ - name: Commit snapshot to main+ env:+ GH_TOKEN: ${{ github.token }}+ FILE: apps/docs/lib/evals/eve-code-benchmark.json+ run: |+ head=$(gh api "repos/$GITHUB_REPOSITORY/git/ref/heads/main" -q .object.sha)+ gh api "repos/$GITHUB_REPOSITORY/contents/$FILE?ref=$head" -H "Accept: application/vnd.github.raw" > "$FILE"+ cp "$FILE" "$RUNNER_TEMP/before.json"+ node .github/scripts/eve-code-docs-snapshot.mjs "$RUNNER_TEMP/bench/report.json" "$FILE"+ if cmp -s "$FILE" "$RUNNER_TEMP/before.json"; then exit 0; fi+ HEAD_OID="$head" node .github/scripts/eve-code-docs-snapshot.mjs --commit-payload "$FILE" > "$RUNNER_TEMP/commit.json"+ gh api graphql --input "$RUNNER_TEMP/commit.json" -q .data.createCommitOnBranch.commit.urldiff --git a/apps/docs/components/geistdocs/eve-code-benchmark.tsx b/apps/docs/components/geistdocs/eve-code-benchmark.tsxnew file mode 100644index 000000000..12f6c6eec--- /dev/null+++ b/apps/docs/components/geistdocs/eve-code-benchmark.tsx@@ -0,0 +1,213 @@+"use client";++import { useState } from "react";++import {+ eveCodeBenchmark,+ rankedHarnesses,+ type EveCodeHarnessScore,+} from "@/lib/evals/eve-code-results";++const REFERENCE = "eve-code";+const COLORS = ["#4f6bed", "#5ed37f", "#e0803f", "#c26bd9", "#d9c24f"];++interface Metric {+ label: string;+ higherIsBetter: boolean;+ value: (score: EveCodeHarnessScore) => number;+ /** Interval drawn around the dot, when the metric has one. */+ range?: (score: EveCodeHarnessScore) => [number, number];+ format: (value: number) => string;+ /** Difference from the reference contender. */+ delta: (value: number, reference: number) => string;+ /** Fixed axis maximum; otherwise fit to the data. */+ max?: number;+}++const signed = (value: number, unit: string) =>+ `${value > 0 ? "+" : value < 0 ? "−" : "±"}${Math.abs(Math.round(value))}${unit}`;+const relative = (value: number, reference: number) =>+ reference === 0 ? "—" : signed((value / reference - 1) * 100, "%");++const METRICS: Record<string, Metric> = {+ correctness: {+ label: "correctness",+ higherIsBetter: true,+ value: (score) => score.resolveRate.estimate,+ range: (score) => [score.resolveRate.low, score.resolveRate.high],+ format: (value) => `${Math.round(value * 100)}%`,+ delta: (value, reference) => signed((value - reference) * 100, " pp"),+ max: 1,+ },+ "input tokens": {+ label: "input tokens",+ higherIsBetter: false,+ value: (score) => score.inputTokens / score.attempts,+ format: (value) =>+ value >= 1_000_000 ? `${(value / 1_000_000).toFixed(1)}M` : `${Math.round(value / 1000)}k`,+ delta: relative,+ },+ latency: {+ label: "latency",+ higherIsBetter: false,+ value: (score) => score.latencyP50Ms,+ range: (score) => [score.latencyP50Ms, score.latencyP90Ms],+ format: (value) => `${Math.round(value / 1000)}s`,+ delta: relative,+ },+};++/** Renders the latest published eve-code results for one eve-bench dataset. */+export const EveCodeBenchmark = ({ dataset }: { dataset: string }) => {+ const [metricKey, setMetricKey] = useState("correctness");+ const results = eveCodeBenchmark.datasets[dataset];+ if (!results) return <p>No published {dataset} results yet.</p>;++ const metric = METRICS[metricKey];+ const rows = rankedHarnesses(results);+ const colors = new Map(+ rows.map((score, index) => [score.harness, COLORS[index % COLORS.length]]),+ );+ const reference = rows.find((score) => score.harness === REFERENCE);+ const max =+ metric.max ??+ niceCeiling(Math.max(...rows.map((score) => metric.range?.(score)[1] ?? metric.value(score))));+ const ticks = [0, 0.25, 0.5, 0.75, 1].map((fraction) => fraction * max);+ const best = (metric.higherIsBetter ? Math.max : Math.min)(...rows.map(metric.value));+ const position = (value: number) => `${(value / max) * 100}%`;+ const runs = new Set(rows.map((score) => score.runUrl)).size;+ const sorted = [...rows].sort((left, right) =>+ metric.higherIsBetter+ ? metric.value(right) - metric.value(left)+ : metric.value(left) - metric.value(right),+ );++ return (+ <figure className="not-prose my-6 overflow-x-auto rounded-xl border border-gray-400 bg-background-100 p-4 text-sm">+ <figcaption className="flex flex-wrap items-baseline gap-x-3 gap-y-1">+ <span className="font-mono font-medium text-gray-1000">+ {results.dataset.name}@{results.dataset.version} · {results.model.id.split("/").at(-1)}+ </span>+ <span className="text-gray-800">+ {rows.length} contenders across {runs} run{runs === 1 ? "" : "s"}+ </span>+ </figcaption>++ <div+ aria-label="Metric"+ className="mt-3 inline-flex rounded-lg border border-gray-400 p-0.5"+ role="tablist"+ >+ {Object.keys(METRICS).map((key) => (+ <button+ aria-selected={key === metricKey}+ className={`rounded-md px-3 py-1 ${key === metricKey ? "bg-gray-200 text-gray-1000" : "text-gray-800 hover:text-gray-1000"}`}+ key={key}+ onClick={() => setMetricKey(key)}+ role="tab"+ type="button"+ >+ {key}+ </button>+ ))}+ </div>++ <div className="mt-4 min-w-[640px]" role="tabpanel">+ <div className="grid grid-cols-[12rem_1fr_6rem_5rem] items-end gap-x-4 pb-1 text-gray-800">+ <span>{metric.higherIsBetter ? "higher is better" : "lower is better"}</span>+ <div className="relative h-5 font-mono text-xs">+ {ticks.map((tick) => (+ <span+ className="absolute -translate-x-1/2"+ key={tick}+ style={{ left: position(tick) }}+ >+ {metric.format(tick)}+ </span>+ ))}+ </div>+ <span className="text-right">{metric.label}</span>+ <span className="text-right">vs ref</span>+ </div>++ {sorted.map((score) => {+ const value = metric.value(score);+ const color = colors.get(score.harness);+ const range = metric.range?.(score);+ const isReference = score.harness === REFERENCE;+ return (+ <div+ className="grid grid-cols-[12rem_1fr_6rem_5rem] items-center gap-x-4 rounded-md py-1.5 hover:bg-gray-100"+ key={score.harness}+ >+ <span className="flex min-w-0 items-center gap-2 font-mono text-gray-1000">+ <span className="size-2.5 shrink-0 rounded-full" style={{ background: color }} />+ <span className="truncate">+ {score.runUrl ? (+ <a href={score.runUrl} title="Open the GitHub Actions run">+ {score.harness}+ {score.version ? `@${score.version}` : ""}+ </a>+ ) : (+ `${score.harness}${score.version ? `@${score.version}` : ""}`+ )}+ </span>+ </span>++ <div className="relative h-5">+ {ticks.map((tick) => (+ <span+ className="absolute inset-y-0 w-px bg-gray-300"+ key={tick}+ style={{ left: position(tick) }}+ />+ ))}+ {range ? (+ <span+ className="absolute top-1/2 h-0.5 -translate-y-1/2 opacity-60"+ style={{+ background: color,+ left: position(range[0]),+ width: `calc(${position(range[1])} - ${position(range[0])})`,+ }}+ />+ ) : null}+ <span+ aria-label={`${score.harness}: ${metric.format(value)}${range ? `, interval ${metric.format(range[0])} to ${metric.format(range[1])}` : ""}`}+ className="absolute top-1/2 size-3.5 -translate-x-1/2 -translate-y-1/2 rounded-full ring-2 ring-background-100 transition-[left] duration-300"+ role="img"+ style={{ background: color, left: position(value) }}+ />+ </div>++ <span+ className={`text-right font-mono ${value === best ? "text-green-700" : "text-gray-1000"}`}+ >+ {metric.format(value)}+ </span>+ <span className="text-right font-mono text-gray-900">+ {isReference || !reference ? "ref" : metric.delta(value, metric.value(reference))}+ </span>+ </div>+ );+ })}+ </div>++ <p className="mt-3 text-gray-700 text-xs">+ {metric.range+ ? metricKey === "latency"+ ? "Dots are median latency; lines extend to p90."+ : "Lines are 95% intervals; overlapping lines are not a measured difference."+ : "Mean input tokens per attempt."}{" "}+ Click a contender to open its run.+ </p>+ </figure>+ );+};++function niceCeiling(value: number): number {+ if (value <= 0) return 1;+ const magnitude = 10 ** Math.floor(Math.log10(value));+ const step = [1, 2, 2.5, 5, 10].find((candidate) => candidate * magnitude >= value) ?? 10;+ return step * magnitude;+}diff --git a/apps/docs/components/geistdocs/mdx-components.tsx b/apps/docs/components/geistdocs/mdx-components.tsxindex 1f5152b2f..5ed5bcbfc 100644--- a/apps/docs/components/geistdocs/mdx-components.tsx+++ b/apps/docs/components/geistdocs/mdx-components.tsx@@ -5,6 +5,7 @@ import { Flag } from "lucide-react";import type { MDXComponents } from "mdx/types";import Link from "next/link";import { AgentRuntimeDiagram } from "./agent-runtime-diagram";+import { EveCodeBenchmark } from "./eve-code-benchmark";const localComponents: MDXComponents = {a: ({ href, ...props }) =>@@ -23,6 +24,7 @@ const localComponents: MDXComponents = {Flag,Folder,AgentRuntimeDiagram,+ EveCodeBenchmark,Step,Steps,};diff --git a/apps/docs/lib/evals/eve-code-benchmark.json b/apps/docs/lib/evals/eve-code-benchmark.jsonnew file mode 100644index 000000000..b65b62b32--- /dev/null+++ b/apps/docs/lib/evals/eve-code-benchmark.json@@ -0,0 +1,84 @@+{+ "schemaVersion": 1,+ "datasets": {+ "deepswe-lean": {+ "dataset": {+ "name": "deepswe-lean",+ "version": "v1"+ },+ "model": {+ "id": "anthropic/claude-sonnet-5.5",+ "released": "2026-09-28",+ "knowledge": "2026-06"+ },+ "attempts": 3,+ "tasks": 12,+ "harnesses": [+ {+ "harness": "eve-code",+ "version": "4d14914",+ "resolved": 24,+ "attempts": 36,+ "resolveRate": {+ "estimate": 0.6666666666666666,+ "low": 0.5277777777777776,+ "high": 0.8055555555555557+ },+ "costUsd": 20.336862000000004,+ "costPerResolvedUsd": 0.8473692500000002,+ "inputTokens": 35312195,+ "outputTokens": 865005,+ "cachedShare": 0.94331136877784,+ "latencyP50Ms": 235438.5,+ "latencyP90Ms": 389674,+ "source": "4d1491477a2e4f220131a3eb857a030c9eec0419",+ "runUrl": "https://github.com/vercel/eve/actions/runs/36775772040",+ "measuredAt": "2026-09-30T21:11:47.181Z"+ },+ {+ "harness": "opencode",+ "version": "1.18.30",+ "resolved": 18,+ "attempts": 36,+ "resolveRate": {+ "estimate": 0.5,+ "low": 0.27777777777777773,+ "high": 0.7222222222222223+ },+ "costUsd": null,+ "costPerResolvedUsd": null,+ "inputTokens": 25457380,+ "outputTokens": 2837675,+ "cachedShare": 0.9378020047624697,+ "latencyP50Ms": 611686.5,+ "latencyP90Ms": 1170330.5,+ "source": "a7b0fb2bf24d57e6bb3613d557c0af53a171d558",+ "runUrl": "https://github.com/vercel/eve/actions/runs/36661642270",+ "measuredAt": "2026-09-30T03:33:15.672Z"+ },+ {+ "harness": "pi",+ "version": "0.85.1",+ "resolved": 26,+ "attempts": 36,+ "resolveRate": {+ "estimate": 0.7222222222222222,+ "low": 0.5,+ "high": 0.9166666666666666+ },+ "costUsd": null,+ "costPerResolvedUsd": null,+ "inputTokens": 27007661,+ "outputTokens": 903678,+ "cachedShare": 0,+ "latencyP50Ms": 266711,+ "latencyP90Ms": 340482.5,+ "source": "a7b0fb2bf24d57e6bb3613d557c0af53a171d558",+ "runUrl": "https://github.com/vercel/eve/actions/runs/36661642270",+ "measuredAt": "2026-09-30T03:33:15.823Z"+ }+ ],+ "generatedAt": "2026-10-03T03:29:12.252Z"+ }+ }+}diff --git a/apps/docs/lib/evals/eve-code-results.ts b/apps/docs/lib/evals/eve-code-results.tsnew file mode 100644index 000000000..977d68a38--- /dev/null+++ b/apps/docs/lib/evals/eve-code-results.ts@@ -0,0 +1,46 @@+import rawSnapshot from "./eve-code-benchmark.json";++/** Written by `.github/scripts/eve-code-docs-snapshot.mjs` from eve-bench reports of `main`. */+export interface EveCodeHarnessScore {+ harness: string;+ /** Package version for released harnesses; short source commit for eve-code. */+ version: string | null;+ resolved: number;+ attempts: number;+ resolveRate: { estimate: number; low: number; high: number };+ costUsd: number | null;+ costPerResolvedUsd: number | null;+ inputTokens: number;+ outputTokens: number;+ cachedShare: number | null;+ latencyP50Ms: number;+ latencyP90Ms: number;+ source: string | null;+ runUrl: string | null;+ measuredAt: string | null;+}++export interface EveCodeDatasetResults {+ dataset: { name: string; version: string };+ model: { id: string; released?: string; knowledge?: string };+ attempts: number;+ tasks: number;+ generatedAt: string;+ harnesses: EveCodeHarnessScore[];+}++export interface EveCodeBenchmarkSnapshot {+ schemaVersion: 1;+ datasets: Record<string, EveCodeDatasetResults>;+}++export const eveCodeBenchmark = rawSnapshot as EveCodeBenchmarkSnapshot;++/** Ranks by resolve rate, then by median latency, matching eve-bench's correctness-first order. */+export function rankedHarnesses(results: EveCodeDatasetResults): EveCodeHarnessScore[] {+ return [...results.harnesses].sort(+ (left, right) =>+ right.resolveRate.estimate - left.resolveRate.estimate ||+ left.latencyP50Ms - right.latencyP50Ms,+ );+}diff --git a/docs/README.md b/docs/README.mdindex c0c065419..b10ae11d6 100644--- a/docs/README.md+++ b/docs/README.md@@ -28,6 +28,7 @@ Important naming note:| Delegate work to a specialist child agent | [Subagents](./subagents/index.mdx) || Run work on a recurring schedule | [Schedules](./schedules.mdx) || Install an existing integration instead of writing one | [Add Integrations](./install-integrations.mdx) |+| Build a coding agent, or compare eve-code benchmarks | [Code Extension](./code-extension.mdx) || Link a Vercel project and deploy to production | [Deploy to Vercel](./guides/deployment/vercel.mdx) || Self-host, or compare hosting strategies | [Deployment](./guides/deployment/overview.md) || Authorize routes, sessions, and per-user access | [Authentication](./guides/auth-and-route-protection.md) |diff --git a/docs/code-extension.mdx b/docs/code-extension.mdxnew file mode 100644index 000000000..b5a8e0382--- /dev/null+++ b/docs/code-extension.mdx@@ -0,0 +1,113 @@+---+title: "Code Extension"+description: "Mount eve's coding extension, eve-code, and see its latest benchmark results against other coding agent harnesses."+---++`eve/extensions/code` (eve-code) turns an eve agent into a coding agent. It ships inside the `eve` package, so you mount it without installing anything else. The extension contributes file-editing and search tools, an authenticated GitHub CLI tool, coding skills, a read-only worker subagent, and sandbox bootstrap helpers.++## Benchmark results++<EveCodeBenchmark dataset="deepswe-lean" />++These results show baseline performance. Each harness uses its standard setup with high reasoning. The results do not show how a harness performs with other models, settings, or tasks.++<details>+<summary>How the benchmark works</summary>++The benchmark uses eve-bench and the DeepSWE-lean dataset. DeepSWE-lean has 12 feature and bug-fix tasks from DeepSWE v1.1.++- All harnesses use the same model and the same tasks.+- Each harness does each task 3 times.+- Each attempt runs in a new Vercel Sandbox.+- Each attempt has a limit of 20 minutes.+- The DeepSWE verifier examines the files that the agent leaves.+- An attempt passes only when the verifier passes.+- A crash or a timeout is a failure.++The line on each correctness row shows the 95% interval. When the lines of two harnesses overlap, the benchmark does not show a difference between them.++</details>++<details>+<summary>How the results update</summary>++1. A maintainer runs the `eve-code` workflow on `main` with the `deepswe-lean` dataset.+2. The workflow runs eve-code, opencode, and pi. It saves each complete result in the eve-bench result store.+3. The workflow writes a summary of the result to `apps/docs/lib/evals/eve-code-benchmark.json`. Then it commits the file to `main`.+4. The next docs deployment shows the new results.++The workflow does not publish incomplete runs, single-task runs, or pull request runs. Results can be older than `main`, because a maintainer starts each run. Select a harness name to open the GitHub Actions run for that result.++</details>++## Mount the extension++Create an extension file in your agent. The file name sets the mount name, which prefixes every contributed tool and subagent ID (for example, `code__grep`):++```ts title="agent/extensions/code.ts"+import code from "eve/extensions/code";++export default code({});+```++`code({})` mounts the extension without GitHub or Vercel authentication. Add the options below when the agent needs them.++## Configure the extension++| Option | Required | Effect |+| ------------------------------ | ------------- | ---------------------------------------------------------------------------------------------- |+| `github.connector` | With `github` | Connector that mints a GitHub token |+| `github.org` | With `github` | Organization whose repositories the `gh` tool can access |+| `github.broker` | With `github` | `broker(sandbox, rules)` callback that installs, then removes, the GitHub credential rules |+| `vercel.connector` | With `vercel` | Connector that requests an app-subject Vercel token |+| `vercel.delivery` | No | `"firewall"` (default) or `"command"` |+| `broker` | No | `broker(sandbox, rules)` callback that merges Vercel credential rules into your network policy |+| `worker.model` | With `worker` | Model for the `worker` subagent. Defaults to `openai/gpt-5.6-terra-fast` |+| `worker.reasoning` | With `worker` | Reasoning level for the `worker` subagent. Defaults to `xhigh` |+| `worker.openaiReasoningEffort` | No | Passed to OpenAI models as `reasoningEffort` |++```ts title="agent/extensions/code.ts"+import code from "eve/extensions/code";++export default code({+ vercel: { connector: "vercel/acme-bot" },+ worker: { model: "openai/gpt-5.6-terra-fast", reasoning: "high" },+});+```++### Credentials stay outside the sandbox++With firewall delivery, Vercel tokens never enter sandbox processes. Firewall delivery without a top-level `broker` requires a sandbox provider that exposes `setNetworkPolicy()`, and fails otherwise. Set `vercel.delivery: "command"` for providers without a mutable network policy.++The `gh` tool requests a token scoped to exactly one repository in `github.org` for each invocation. The sandbox process receives a placeholder `GH_TOKEN`, and the firewall exchanges it on matching GitHub requests. Your `github.broker` callback receives the header-transform rules for the command, then `null` to remove the lease. The callback must preserve your other network rules. GitHub authentication has no command-delivery option.++To use a personal access token or another credential provider, omit the connector and call `authenticateGitHub` or `authenticateVercel` from `eve/extensions/code/sandbox` in your own sandbox lifecycle. These helpers authenticate commands you run yourself; they do not configure the `gh` tool.++## Prepare the sandbox++Install the CLI tooling in your sandbox environment's `prepare` callback:++```ts title="agent/sandbox.ts"+import { defineSandbox } from "eve/sandbox";+import { VercelSandbox } from "eve/sandbox/vercel";+import { installCodeTooling } from "eve/extensions/code/sandbox";++export const environment = VercelSandbox.environment({+ prepare: async (sandbox) => {+ await installCodeTooling(sandbox, { vercel: true });+ },+});++export default defineSandbox(() => environment.open());+```++Preparation installs `gh`, wrappers for `gh`, `vc`, and `gh-signed-commit`, and TypeScript diagnostics. For repositories that require verified signatures, stage your changes and commit with `gh-signed-commit`.++eve derives the prepared environment generation from the sandbox file and environment options, not from imported helpers. Upgrading eve alone does not rebuild an existing prepared environment.++## What to read next++- [Extensions](/docs/extensions): mount, configure, and override extension packages+- [Sandbox](/docs/sandbox): environments, preparation, and network policy+- [Subagents](/docs/subagents): how declared subagents run and inherit configuration+- [Skills](/docs/skills): how the model loads skills on demanddiff --git a/docs/extensions.md b/docs/extensions.mdindex bc5dcf447..a039bd1ac 100644--- a/docs/extensions.md+++ b/docs/extensions.md@@ -370,6 +370,7 @@ At build time, eve checks the extension's generated capability metadata. If the## What to read next- [Integrations](/integrations): browse ready-to-install extensions using the Extensions filter+- [Code extension](/docs/code-extension): mount eve-code, the coding extension that ships in `eve`, and see its benchmark results- [Tools](/docs/tools): static tools, approval, and tool output- [Dynamic capabilities](/docs/guides/dynamic-capabilities): dynamic connections, tools, skills, and instructions- [Instructions](/docs/instructions): static and TypeScript instructionsdiff --git a/docs/meta.json b/docs/meta.jsonindex 0dcbf7570..44e421fa0 100644--- a/docs/meta.json+++ b/docs/meta.json@@ -20,6 +20,7 @@"---Integrate---","install-integrations","extensions",+ "code-extension","protocols/acp","guides/client","guides/frontend",diff --git a/packages/eve/scripts/copy-docs.mjs b/packages/eve/scripts/copy-docs.mjsindex 27b449435..51873bb69 100644--- a/packages/eve/scripts/copy-docs.mjs+++ b/packages/eve/scripts/copy-docs.mjs@@ -1,4 +1,4 @@-import { cp, rm } from "node:fs/promises";+import { cp, readFile, rm, writeFile } from "node:fs/promises";import { join } from "node:path";import { fileURLToPath } from "node:url";@@ -14,3 +14,42 @@ const oldDistDocsDir = join(packageRoot, "dist", "docs");await rm(packageDocsDir, { recursive: true, force: true });await rm(oldDistDocsDir, { recursive: true, force: true });await cp(join(monorepoRoot, "docs"), packageDocsDir, { recursive: true });++// Agents read the raw file, where the site's <EveCodeBenchmark /> component+// would render nothing, so inline the same snapshot as a Markdown table.+const benchmarkPage = join(packageDocsDir, "code-extension.mdx");+const page = await readFile(benchmarkPage, "utf8").catch((error) => {+ if (error.code === "ENOENT") return null;+ throw error;+});+if (page !== null) {+ const snapshot = JSON.parse(+ await readFile(join(monorepoRoot, "apps/docs/lib/evals/eve-code-benchmark.json"), "utf8"),+ );+ await writeFile(+ benchmarkPage,+ page.replace(/<EveCodeBenchmark dataset="([^"]+)" \/>/gu, (_, dataset) =>+ benchmarkTable(snapshot.datasets[dataset], dataset),+ ),+ );+}++function benchmarkTable(results, dataset) {+ if (!results) return `No published ${dataset} results yet.`;+ const pct = (value) => `${Math.round(value * 100)}%`;+ const rows = [...results.harnesses]+ .sort(+ (a, b) => b.resolveRate.estimate - a.resolveRate.estimate || a.latencyP50Ms - b.latencyP50Ms,+ )+ .map(+ (h) =>+ `| \`${h.harness}\` | ${pct(h.resolveRate.estimate)} (${h.resolved}/${h.attempts}) | ${pct(h.resolveRate.low)}–${pct(h.resolveRate.high)} | ${(h.latencyP50Ms / 1000).toFixed(1)}s | ${(h.latencyP90Ms / 1000).toFixed(1)}s | ${h.measuredAt?.slice(0, 10) ?? "—"} |`,+ );+ return [+ `${results.tasks} tasks × ${results.attempts} attempts on \`${results.model.id}\`. Snapshot updated ${results.generatedAt.slice(0, 10)}.`,+ "",+ "| Harness | Resolved | 95% interval | Median latency | p90 latency | Measured |",+ "| --- | --- | --- | --- | --- | --- |",+ ...rows,+ ].join("\n");+}