Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

feat: publish authoring benchmark results (#2057)

Colton Padden4f78a2d336a7
diff --git a/.gitignore b/.gitignore
index f06caebc0..305d74c08 100644
--- a/.gitignore
+++ b/.gitignore
@@ -51,6 +51,7 @@ Thumbs.db
# IDE/editor
.codex/
+.githooks/
.vscode/
.idea/
diff --git a/apps/benchmarks/README.md b/apps/benchmarks/README.md
index 752377095..54e30db5a 100644
--- a/apps/benchmarks/README.md
+++ b/apps/benchmarks/README.md
@@ -13,6 +13,7 @@ Git does not ignore:
pnpm benchmark author-000-imessage
pnpm benchmark
pnpm benchmark author-000-imessage --runs 3
+pnpm benchmark author-000-imessage --treatment baseline
pnpm benchmark author-000-imessage --dry
pnpm benchmark author-000-imessage --verbose
```
@@ -37,10 +38,35 @@ commits do not need to be pushed. Dependency downloads are cached by lockfile, s
changes reuse the prepared pnpm store. For one eval and one run, `--verbose` streams setup
phases, assistant text, tool calls, grading, and build progress.
+Local runs use the `guided` treatment by default, which keeps the `AGENTS.md` and aliases generated
+by `eve init`. Pass `--treatment baseline` to remove those files before the coding agent starts.
+
Results are written under `apps/benchmarks/results/`. Each run includes the transcript,
grader output, summary, and copied project files. Vercel Sandbox and AI Gateway credentials are
required.
+## Publish canonical results
+
+Canonical publication compares the `baseline` and `guided` treatments with the same eve revision,
+model, harness, cases, and graders. It requires a clean working tree and defaults to `origin/main`:
+
+```sh
+pnpm benchmark:publish --dry
+pnpm benchmark:publish
+pnpm benchmark:publish --revision <commit>
+```
+
+Pass `--allow-dirty` only for a local, noncanonical run. It bypasses the clean-tree check, so its
+results cannot be reproduced from committed source and should not be committed as published data.
+
+Each cell runs three times. Completed cells are memoized by `@vercel/agent-eval`; pass `--force` only
+when every cell should run again. A successful run writes aggregate results to
+`apps/docs/lib/evals/benchmark-results.json`. The public file contains outcomes and timing, not
+transcripts, generated files, command logs, or synthetic world events.
+
+Changed and newly added cases export as unavailable until they run. The exporter does not carry an
+older measurement forward as current.
+
## Add a case
Create the next `evals/author-NNN-*` directory with:
diff --git a/apps/benchmarks/evals/author-000-imessage/CASE.ts b/apps/benchmarks/evals/author-000-imessage/CASE.ts
index 278d939a9..106b32061 100644
--- a/apps/benchmarks/evals/author-000-imessage/CASE.ts
+++ b/apps/benchmarks/evals/author-000-imessage/CASE.ts
@@ -1,18 +1,14 @@
import { defineAuthoringCase, simpleProject } from "../../lib/authoring-case.js";
import { imessageSetup } from "../../lib/setups/imessage.js";
-const PHONE_NUMBER = "+447700900123";
-const PHONE_NUMBER_QUESTION =
- /(?:what|which|provide|share|send|need)[^.!?\n]{0,100}(?:phone|imessage)[^.!?\n]{0,50}number|(?:phone|imessage)[^.!?\n]{0,50}number[^.!?\n]{0,100}(?:use|receive|register|provide|share|send|need)|(?:blocked|waiting)[^.!?\n]{0,120}(?:phone|imessage)[^.!?\n]{0,50}number/i;
+const PHONE_NUMBER = "+15551234567";
export default defineAuthoringCase({
startingPoint: simpleProject,
setup: imessageSetup,
async interact({ send }) {
- const firstTurn = await send(
- "Let me talk to this agent via iMessage. Configure it locally, but do not deploy it.",
- );
- if (!PHONE_NUMBER_QUESTION.test(firstTurn.text)) {
+ const firstTurn = await send("Let me talk to this agent via iMessage.");
+ if (!/phone number|imessage number|number should/i.test(firstTurn.text)) {
throw new Error(
`Expected the agent to ask for the user's phone number on its first turn. Received: ${JSON.stringify(firstTurn.text)}`,
);
diff --git a/apps/benchmarks/evals/author-000-imessage/EVAL.ts b/apps/benchmarks/evals/author-000-imessage/EVAL.ts
index 5fef3c4be..5a16d2416 100644
--- a/apps/benchmarks/evals/author-000-imessage/EVAL.ts
+++ b/apps/benchmarks/evals/author-000-imessage/EVAL.ts
@@ -1,9 +1,8 @@
import { existsSync, readFileSync } from "node:fs";
-import { join } from "node:path";
import { expect, test } from "vitest";
-import { authoringEval, workspace } from "./grader.js";
+import { authoringEval } from "./grader.js";
const { commands, transcript, worldEvents } = authoringEval();
const commandLog = commands.join("\n");
@@ -15,7 +14,7 @@ test("installs the discovered iMessage registry item through the headless setup
test("asks for and uses the phone number from the follow-up turn", () => {
expect(transcript.some((entry) => /phone number/i.test(entry.content))).toBe(true);
- expect(transcript.some((entry) => entry.content.includes("+447700900123"))).toBe(true);
+ expect(transcript.some((entry) => entry.content.includes("+15551234567"))).toBe(true);
});
test("completes the synthetic provider setup decision tree", () => {
@@ -23,11 +22,11 @@ test("completes the synthetic provider setup decision tree", () => {
expect.arrayContaining(["project.created", "phone.registered", "setup.completed"]),
);
const registration = worldEvents.find((event) => event.type === "phone.registered");
- expect(registration?.data?.phoneNumber).toBe("+447700900123");
+ expect(registration?.data?.phoneNumber).toBe("+15551234567");
});
test("creates an iMessage channel and leaves the project valid", () => {
- const channelPath = join(workspace, "agent/channels/imessage.ts");
+ const channelPath = "agent/channels/imessage.ts";
expect(existsSync(channelPath)).toBe(true);
expect(readFileSync(channelPath, "utf8")).toContain("photonIMessageChannel");
});
diff --git a/apps/benchmarks/evals/author-001-weather-tool/EVAL.ts b/apps/benchmarks/evals/author-001-weather-tool/EVAL.ts
index 41703f74d..e268d7163 100644
--- a/apps/benchmarks/evals/author-001-weather-tool/EVAL.ts
+++ b/apps/benchmarks/evals/author-001-weather-tool/EVAL.ts
@@ -1,11 +1,8 @@
import { existsSync, readFileSync } from "node:fs";
-import { join } from "node:path";
import { expect, test } from "vitest";
-import { workspace } from "./grader.js";
-
-const toolPath = join(workspace, "agent/tools/get_weather.ts");
+const toolPath = "agent/tools/get_weather.ts";
test("creates a filesystem-named weather tool", () => {
expect(existsSync(toolPath)).toBe(true);
diff --git a/apps/benchmarks/evals/author-002-new-project/CASE.ts b/apps/benchmarks/evals/author-002-new-project/CASE.ts
index b6e64824e..0418635f0 100644
--- a/apps/benchmarks/evals/author-002-new-project/CASE.ts
+++ b/apps/benchmarks/evals/author-002-new-project/CASE.ts
@@ -4,7 +4,7 @@ export default defineAuthoringCase({
startingPoint: emptyProject,
async interact({ send }) {
await send(
- "Create a new eve project in this directory for a concise travel assistant called Wayfinder. It should use the default model and be ready to build.",
+ "Create a new eve project directly in the current working directory for a concise travel assistant called Wayfinder. Do not create a subdirectory: initialize the current directory (for example, with `eve init .`). It should use the default model and be ready to build.",
);
},
});
diff --git a/apps/benchmarks/evals/author-002-new-project/EVAL.ts b/apps/benchmarks/evals/author-002-new-project/EVAL.ts
index b54c239cc..cc5d10799 100644
--- a/apps/benchmarks/evals/author-002-new-project/EVAL.ts
+++ b/apps/benchmarks/evals/author-002-new-project/EVAL.ts
@@ -1,18 +1,15 @@
import { existsSync, readFileSync } from "node:fs";
-import { join } from "node:path";
import { expect, test } from "vitest";
-import { subjectDefaultAgentModel, workspace } from "./grader.js";
-
-const projectPath = (path: string) => join(workspace, path);
+import { subjectDefaultAgentModel } from "./grader.js";
test("creates a complete eve project in place", () => {
- expect(existsSync(projectPath("agent/agent.ts"))).toBe(true);
- expect(existsSync(projectPath("agent/channels/eve.ts"))).toBe(true);
- expect(existsSync(projectPath("agent/instructions.md"))).toBe(true);
+ expect(existsSync("agent/agent.ts")).toBe(true);
+ expect(existsSync("agent/channels/eve.ts")).toBe(true);
+ expect(existsSync("agent/instructions.md")).toBe(true);
- const packageJson = JSON.parse(readFileSync(projectPath("package.json"), "utf8")) as {
+ const packageJson = JSON.parse(readFileSync("package.json", "utf8")) as {
dependencies?: Record<string, string>;
scripts?: Record<string, string>;
};
@@ -21,9 +18,9 @@ test("creates a complete eve project in place", () => {
});
test("authors the requested identity without replacing the default model", () => {
- expect(readFileSync(projectPath("agent/instructions.md"), "utf8")).toMatch(/Wayfinder/i);
- expect(readFileSync(projectPath("agent/instructions.md"), "utf8")).toMatch(/travel/i);
- expect(readFileSync(projectPath("agent/agent.ts"), "utf8")).toContain(
+ expect(readFileSync("agent/instructions.md", "utf8")).toMatch(/Wayfinder/i);
+ expect(readFileSync("agent/instructions.md", "utf8")).toMatch(/travel/i);
+ expect(readFileSync("agent/agent.ts", "utf8")).toContain(
`model: "${subjectDefaultAgentModel()}"`,
);
});
diff --git a/apps/benchmarks/lib/authoring-case.ts b/apps/benchmarks/lib/authoring-case.ts
index 0a32d0171..bd9481e85 100644
--- a/apps/benchmarks/lib/authoring-case.ts
+++ b/apps/benchmarks/lib/authoring-case.ts
@@ -30,7 +30,7 @@ export interface AuthoringSetup {
export interface AuthoringTurn {
readonly text: string;
- readonly toolCalls: ReadonlyArray<{ input: unknown }>;
+ readonly toolCalls: ReadonlyArray<{ name: string; input: unknown }>;
}
export interface AuthoringInteractionContext {
@@ -46,12 +46,12 @@ export interface AuthoringCase {
}
export const emptyProject: AuthoringStartingPoint = {
- id: "empty",
+ id: "empty-v2",
workspace: "empty",
};
export const simpleProject: AuthoringStartingPoint = {
- id: "simple",
+ id: "simple-v2",
workspace: "scaffolded",
};
diff --git a/apps/benchmarks/lib/benchmark-config.ts b/apps/benchmarks/lib/benchmark-config.ts
new file mode 100644
index 000000000..7d62ada2a
--- /dev/null
+++ b/apps/benchmarks/lib/benchmark-config.ts
@@ -0,0 +1,26 @@
+export const AUTHORING_MODEL = "claude-sonnet-4-6";
+
+export const authoringTreatments = ["baseline", "guided"] as const;
+
+export type AuthoringTreatment = (typeof authoringTreatments)[number];
+
+export const publishedBenchmark = {
+ groupId: "claude-sonnet-4-6-claude-code",
+ model: AUTHORING_MODEL,
+ modelDisplayName: "Claude Sonnet 4.6",
+ harness: "Claude Code",
+ runs: 3,
+} as const;
+
+export function publishedExperimentId(treatment: AuthoringTreatment): string {
+ return `${publishedBenchmark.groupId}--${treatment}`;
+}
+
+export function parseAuthoringTreatment(value: string): AuthoringTreatment {
+ if (authoringTreatments.includes(value as AuthoringTreatment)) {
+ return value as AuthoringTreatment;
+ }
+ throw new Error(
+ `Unknown treatment ${JSON.stringify(value)}. Expected one of: ${authoringTreatments.join(", ")}.`,
+ );
+}
diff --git a/apps/benchmarks/lib/dependency-sandbox.ts b/apps/benchmarks/lib/dependency-sandbox.ts
index dec75c7d9..b2b9da024 100644
--- a/apps/benchmarks/lib/dependency-sandbox.ts
+++ b/apps/benchmarks/lib/dependency-sandbox.ts
@@ -4,8 +4,7 @@ import { Sandbox } from "@vercel/sandbox";
import { SOURCE_ARCHIVE_PATH, SOURCE_ROOT } from "./paths.js";
-const dependencySnapshots = new Map<string, Promise<string>>();
-const subjectSnapshots = new Map<string, Promise<string>>();
+const snapshots = new Map<string, Promise<string>>();
export function createDependencyCachedSandbox(options: {
readonly archive: Uint8Array;
@@ -14,53 +13,35 @@ export function createDependencyCachedSandbox(options: {
readonly env: Readonly<Record<string, string>>;
readonly log: (message: string) => void;
}): HarnessV1SandboxProvider {
- const sessionProvider = (snapshotId?: string) =>
- snapshotId === undefined
- ? createVercelSandbox({
- runtime: "node24",
- ports: [...options.ports],
- timeout: 15 * 60_000,
- env: { ...options.env },
- networkPolicy: "allow-all",
- })
- : createVercelSandbox({
- source: { type: "snapshot", snapshotId },
- ports: [...options.ports],
- timeout: 15 * 60_000,
- env: { ...options.env },
- networkPolicy: "allow-all",
- });
-
+ let provider: Promise<HarnessV1SandboxProvider> | undefined;
+ const resolveProvider = () => {
+ options.log("[setup] preparing dependency cache");
+ return (provider ??= dependencySnapshot(
+ options.archive,
+ options.dependencyDigest,
+ options.log,
+ ).then((snapshotId) =>
+ createVercelSandbox({
+ source: { type: "snapshot", snapshotId },
+ ports: [...options.ports],
+ timeout: 15 * 60_000,
+ env: { ...options.env },
+ networkPolicy: "allow-all",
+ }),
+ ));
+ };
return {
specificationVersion: "harness-sandbox-v1",
providerId: "eve-benchmark-vercel",
- async createSession(request = {}) {
- if (request.identity === undefined || request.onFirstCreate === undefined) {
- return sessionProvider().createSession(request);
- }
- options.log("[setup] preparing dependency cache");
- const dependencySnapshotId = await dependencySnapshot(
- options.archive,
- options.dependencyDigest,
- options.log,
- );
- const subjectSnapshotId = await subjectSnapshot(
- dependencySnapshotId,
- request.identity,
- options.env,
- request.onFirstCreate,
- request.abortSignal,
- );
- return sessionProvider(subjectSnapshotId).createSession({
- abortSignal: request.abortSignal,
- });
+ async createSession(options) {
+ return (await resolveProvider()).createSession(options);
},
- async resumeSession(request) {
- const provider = sessionProvider();
- if (provider.resumeSession === undefined) {
+ async resumeSession(options) {
+ const resolved = await resolveProvider();
+ if (resolved.resumeSession === undefined) {
throw new Error("Vercel Sandbox does not support session resume.");
}
- return provider.resumeSession(request);
+ return resolved.resumeSession(options);
},
};
}
@@ -70,11 +51,11 @@ function dependencySnapshot(
digest: string,
log: (message: string) => void,
): Promise<string> {
- const name = `eve-benchmark-dependencies-${digest.slice(0, 24)}`;
- let snapshot = dependencySnapshots.get(name);
+ const name = `eve-benchmark-dependencies-v4-${digest.slice(0, 24)}`;
+ let snapshot = snapshots.get(name);
if (snapshot !== undefined) return snapshot;
snapshot = createDependencySnapshot(name, archive, log);
- dependencySnapshots.set(name, snapshot);
+ snapshots.set(name, snapshot);
return snapshot;
}
@@ -83,7 +64,6 @@ async function createDependencySnapshot(
archive: Uint8Array,
log: (message: string) => void,
): Promise<string> {
- let created = false;
const sandbox = await Sandbox.getOrCreate({
name,
runtime: "node24",
@@ -91,11 +71,10 @@ async function createDependencySnapshot(
persistent: true,
snapshotExpiration: 0,
networkPolicy: "allow-all",
- async onCreate(current) {
- created = true;
+ async onCreate(created) {
log("[setup] fetching workspace dependencies");
- await current.writeFiles([{ path: SOURCE_ARCHIVE_PATH, content: archive }]);
- const command = await current.runCommand("bash", [
+ await created.writeFiles([{ path: SOURCE_ARCHIVE_PATH, content: archive }]);
+ const command = await created.runCommand("bash", [
"-lc",
`mkdir -p ${SOURCE_ROOT} && tar -xzf ${SOURCE_ARCHIVE_PATH} -C ${SOURCE_ROOT} && npm install --global [email protected] [email protected] && cd ${SOURCE_ROOT} && pnpm fetch --frozen-lockfile`,
]);
@@ -106,60 +85,8 @@ async function createDependencySnapshot(
}
},
});
- if (!created && sandbox.currentSnapshotId !== undefined) return sandbox.currentSnapshotId;
- return stopWithSnapshot(sandbox, "Dependency");
-}
-
-function subjectSnapshot(
- dependencySnapshotId: string,
- identity: string,
- env: Readonly<Record<string, string>>,
- bootstrap: NonNullable<Parameters<HarnessV1SandboxProvider["createSession"]>[0]>["onFirstCreate"],
- abortSignal?: AbortSignal,
-): Promise<string> {
- const name = `eve-benchmark-subject-${identity}`;
- let snapshot = subjectSnapshots.get(name);
- if (snapshot !== undefined) return snapshot;
- snapshot = createSubjectSnapshot(name, dependencySnapshotId, env, bootstrap, abortSignal);
- subjectSnapshots.set(name, snapshot);
- return snapshot;
-}
-
-async function createSubjectSnapshot(
- name: string,
- dependencySnapshotId: string,
- env: Readonly<Record<string, string>>,
- bootstrap: NonNullable<Parameters<HarnessV1SandboxProvider["createSession"]>[0]>["onFirstCreate"],
- abortSignal?: AbortSignal,
-): Promise<string> {
- let created = false;
- const sandbox = await Sandbox.getOrCreate({
- name,
- source: { type: "snapshot", snapshotId: dependencySnapshotId },
- timeout: 15 * 60_000,
- env: { ...env },
- persistent: true,
- snapshotExpiration: 0,
- networkPolicy: "allow-all",
- signal: abortSignal,
- async onCreate(current) {
- created = true;
- const provider = createVercelSandbox({ sandbox: current });
- const session = await provider.createSession({ abortSignal });
- await bootstrap!(session.restricted(), { abortSignal });
- },
- });
- if (!created && sandbox.currentSnapshotId !== undefined) return sandbox.currentSnapshotId;
- return stopWithSnapshot(sandbox, "Subject", abortSignal);
-}
-
-async function stopWithSnapshot(
- sandbox: Sandbox,
- label: string,
- signal?: AbortSignal,
-): Promise<string> {
- const stopped = await sandbox.stop(signal === undefined ? undefined : { signal });
- const snapshotId = stopped.snapshot?.id ?? sandbox.currentSnapshotId;
- if (snapshotId === undefined) throw new Error(`${label} snapshot was not published.`);
- return snapshotId;
+ if (sandbox.currentSnapshotId !== undefined) return sandbox.currentSnapshotId;
+ const stopped = await sandbox.stop();
+ if (stopped.snapshot?.id === undefined) throw new Error("Dependency snapshot was not published.");
+ return stopped.snapshot.id;
}
diff --git a/apps/benchmarks/lib/experiment.ts b/apps/benchmarks/lib/experiment.ts
index 2cf94eabb..d00b0cf6e 100644
--- a/apps/benchmarks/lib/experiment.ts
+++ b/apps/benchmarks/lib/experiment.ts
@@ -1,18 +1,20 @@
import type { ExperimentConfig } from "@vercel/agent-eval";
import { registerAgent } from "@vercel/agent-eval";
+import type { AuthoringTreatment } from "./benchmark-config.js";
+import { AUTHORING_MODEL } from "./benchmark-config.js";
import { createAuthoringAgent } from "./harness-agent.js";
-import { AUTHORING_MODEL } from "./paths.js";
export function authoringExperiment(options: {
readonly archive: Uint8Array;
readonly digest: string;
readonly dependencyDigest: string;
readonly runs?: number;
+ readonly treatment: AuthoringTreatment;
readonly verbose?: boolean;
}): ExperimentConfig {
const agent = createAuthoringAgent({
- name: `eve-authoring-harness-${options.digest.slice(0, 12)}`,
+ name: `eve-authoring-harness-${options.digest.slice(0, 12)}-${options.treatment}`,
archive: options.archive,
digest: options.digest,
dependencyDigest: options.dependencyDigest,
@@ -24,10 +26,13 @@ export function authoringExperiment(options: {
evals: process.env.EVE_BENCHMARK_EVAL ?? "*",
scripts: ["typecheck", "build"],
runs: options.runs ?? 1,
- earlyExit: options.runs === undefined,
+ earlyExit: false,
timeout: 900,
sandbox: "vercel",
copyFiles: "changed",
- agentOptions: { agentsMd: true, verbose: options.verbose ?? false },
+ agentOptions: {
+ agentsMd: options.treatment === "guided",
+ verbose: options.verbose ?? false,
+ },
};
}
diff --git a/apps/benchmarks/lib/grader.ts b/apps/benchmarks/lib/grader.ts
index 0bfa2474e..ecdd0defb 100644
--- a/apps/benchmarks/lib/grader.ts
+++ b/apps/benchmarks/lib/grader.ts
@@ -1,6 +1,6 @@
import { existsSync, readFileSync } from "node:fs";
-import { AGENT_EVAL_DIRECTORY, SOURCE_ROOT, WORKSPACE_ENV, WORLD_EVENTS_PATH } from "./paths.js";
+import { AGENT_EVAL_DIRECTORY, SOURCE_ROOT, WORLD_EVENTS_PATH } from "./paths.js";
import type { AuthoringTranscriptEntry, AuthoringWorldEvent } from "./protocol.js";
interface AgentEvalResults {
@@ -13,9 +13,6 @@ interface AgentEvalResults {
};
}
-export const workspace = process.env[WORKSPACE_ENV];
-if (workspace === undefined) throw new Error(`${WORKSPACE_ENV} is required.`);
-
export interface AuthoringEvalResult {
readonly commands: ReadonlyArray<string>;
readonly transcript: ReadonlyArray<AuthoringTranscriptEntry>;
diff --git a/apps/benchmarks/lib/harness-agent.ts b/apps/benchmarks/lib/harness-agent.ts
index bb5f0b07e..1737d31e0 100644
--- a/apps/benchmarks/lib/harness-agent.ts
+++ b/apps/benchmarks/lib/harness-agent.ts
@@ -310,12 +310,21 @@ async function streamTurn(
if (verbose && lineOpen) process.stdout.write("\n");
return {
text: await result.text,
- toolCalls: await result.toolCalls,
+ toolCalls: authoringToolCalls(await result.toolCalls),
elapsedMs: performance.now() - startedAt,
steps,
};
}
+function authoringToolCalls(
+ toolCalls: ReadonlyArray<{ name?: string; toolName?: string; input: unknown }>,
+): ReadonlyArray<{ name: string; input: unknown }> {
+ return toolCalls.flatMap((call) => {
+ const name = call.toolName ?? call.name;
+ return name === undefined ? [] : [{ name, input: call.input }];
+ });
+}
+
function formatToolCall(name: string, input: unknown): string {
if (typeof input === "object" && input !== null) {
const command = (input as { command?: unknown }).command;
diff --git a/apps/benchmarks/lib/load-authoring-case.test.mjs b/apps/benchmarks/lib/load-authoring-case.test.mjs
new file mode 100644
index 000000000..05699f080
--- /dev/null
+++ b/apps/benchmarks/lib/load-authoring-case.test.mjs
@@ -0,0 +1,22 @@
+import assert from "node:assert/strict";
+import { readdir } from "node:fs/promises";
+import { dirname, resolve } from "node:path";
+import test from "node:test";
+import { fileURLToPath } from "node:url";
+
+import { createJiti } from "jiti";
+
+const libRoot = dirname(fileURLToPath(import.meta.url));
+const evalsRoot = resolve(libRoot, "../evals");
+const jiti = createJiti(import.meta.url, { interopDefault: true, moduleCache: false });
+const { loadAuthoringCase } = await jiti.import(resolve(libRoot, "load-authoring-case.ts"));
+
+for (const entry of await readdir(evalsRoot, { withFileTypes: true })) {
+ if (!entry.isDirectory()) continue;
+
+ test(`loads ${entry.name}/CASE.ts`, async () => {
+ const authoringCase = await loadAuthoringCase(resolve(evalsRoot, entry.name));
+ assert.equal(typeof authoringCase.interact, "function");
+ assert.ok(authoringCase.startingPoint);
+ });
+}
diff --git a/apps/benchmarks/lib/load-authoring-case.ts b/apps/benchmarks/lib/load-authoring-case.ts
new file mode 100644
index 000000000..a652953e4
--- /dev/null
+++ b/apps/benchmarks/lib/load-authoring-case.ts
@@ -0,0 +1,23 @@
+import { createJiti } from "jiti";
+
+import type { AuthoringCase } from "./authoring-case.js";
+
+export async function loadAuthoringCase(fixturePath: string): Promise<AuthoringCase> {
+ const jiti = createJiti(import.meta.url, { interopDefault: true, moduleCache: false });
+ let loaded: unknown = await jiti.import(`${fixturePath}/CASE.ts`);
+ while (!isAuthoringCase(loaded) && hasDefaultExport(loaded)) loaded = loaded.default;
+ if (!isAuthoringCase(loaded)) {
+ throw new Error(`${fixturePath}/CASE.ts must export an authoring case as default.`);
+ }
+ return loaded;
+}
+
+function isAuthoringCase(value: unknown): value is AuthoringCase {
+ if (typeof value !== "object" || value === null) return false;
+ const candidate = value as Partial<AuthoringCase>;
+ return candidate.startingPoint !== undefined && typeof candidate.interact === "function";
+}
+
+function hasDefaultExport(value: unknown): value is { default: unknown } {
+ return typeof value === "object" && value !== null && "default" in value;
+}
diff --git a/apps/benchmarks/lib/protocol.ts b/apps/benchmarks/lib/protocol.ts
index a2762fb46..8c87ee4ae 100644
--- a/apps/benchmarks/lib/protocol.ts
+++ b/apps/benchmarks/lib/protocol.ts
@@ -1,6 +1,10 @@
export interface AuthoringTranscriptEntry {
readonly role: "user" | "assistant";
readonly content: string;
+ readonly toolCalls?: ReadonlyArray<{
+ readonly name: string;
+ readonly input: unknown;
+ }>;
}
export interface AuthoringWorldEvent {
diff --git a/apps/benchmarks/lib/setups/imessage.ts b/apps/benchmarks/lib/setups/imessage.ts
index 992c2a882..91a5f7a49 100644
--- a/apps/benchmarks/lib/setups/imessage.ts
+++ b/apps/benchmarks/lib/setups/imessage.ts
@@ -6,11 +6,11 @@ const REGISTRY_PORT = 4173;
const REGISTRY_URL = `http://127.0.0.1:${REGISTRY_PORT}`;
export const imessageSetup: AuthoringSetup = {
- id: "imessage",
+ id: "imessage-v1",
ports: [REGISTRY_PORT],
environment: { EVE_DEV_OFFICIAL_REGISTRY_URL: REGISTRY_URL },
async onBootstrap({ run, artifactsRoot, write }) {
- const setupRoot = `${artifactsRoot}/photon-setup`;
+ const setupRoot = `${artifactsRoot}/mock-imessage-setup`;
await Promise.all(
["authoring-world.mjs", "cli.mjs", "package.json"].map((file) =>
write(
@@ -19,29 +19,29 @@ export const imessageSetup: AuthoringSetup = {
),
),
);
- await run(`pnpm add ${setupRoot}`);
+ await run(`pnpm add ./${setupRoot}`);
await run(`mkdir -p ${artifactsRoot}/registry/channel`);
const channel = {
$schema: "https://ui.shadcn.com/schema/registry-item.json",
name: "channel/photon-imessage",
title: "Photon iMessage",
- description: "Connect an eve agent to iMessage through Photon.",
+ description: "Connect an eve agent to iMessage through a deterministic provider setup flow.",
files: [
{
path: "registry/channels/photon-imessage.ts",
type: "registry:file",
target: "agent/channels/imessage.ts",
content:
- 'import { photonIMessageChannel } from "eve/channels/photon";\n\nexport default photonIMessageChannel({ credentials: async () => ({ projectId: "photon-project", projectSecret: "photon-project-secret" }) });\n',
+ 'import { photonIMessageChannel } from "eve/channels/photon";\n\nexport default photonIMessageChannel({ credentials: async () => ({ projectId: "mock-imessage-project", projectSecret: "mock-imessage-secret" }) });\n',
},
],
meta: {
eve: {
setup: {
- command: "photon-setup",
- package: "@photon-ai/setup",
- bin: "photon-setup",
+ command: "mock-imessage-setup",
+ package: "@eve-internal/mock-imessage-setup",
+ bin: "mock-imessage-setup",
args: [],
},
},
@@ -50,7 +50,7 @@ export const imessageSetup: AuthoringSetup = {
};
const registry = {
$schema: "https://ui.shadcn.com/schema/registry.json",
- name: "eve-official",
+ name: "eve-authoring-eval",
homepage: REGISTRY_URL,
items: [
{
@@ -68,9 +68,13 @@ export const imessageSetup: AuthoringSetup = {
write(`${artifactsRoot}/registry/channel/photon-imessage.json`, JSON.stringify(channel)),
]);
},
- async onSession({ run, artifactsRoot }) {
+ async onSession({ run, artifactsRoot, write }) {
await run(
`nohup python3 -m http.server ${REGISTRY_PORT} --directory ${artifactsRoot}/registry >${artifactsRoot}/registry.log 2>&1 </dev/null &`,
);
+ await write(
+ ".claude/settings.json",
+ JSON.stringify({ env: { EVE_DEV_OFFICIAL_REGISTRY_URL: REGISTRY_URL } }),
+ );
},
};
diff --git a/apps/benchmarks/lib/setups/mock-imessage-setup/cli.mjs b/apps/benchmarks/lib/setups/mock-imessage-setup/cli.mjs
index 4e0894944..c95d6bcb4 100644
--- a/apps/benchmarks/lib/setups/mock-imessage-setup/cli.mjs
+++ b/apps/benchmarks/lib/setups/mock-imessage-setup/cli.mjs
@@ -6,7 +6,8 @@ import { parseArgs } from "node:util";
import { authoringStatePath, recordAuthoringEvent } from "./authoring-world.mjs";
const PROTOCOL_VERSION = 2;
-const STATE_FILE = authoringStatePath("photon-state");
+const STATE_FILE = authoringStatePath("mock-imessage-state");
+const EXPECTED_PHONE = process.env.EVE_AUTHORING_PHONE_NUMBER ?? "+15551234567";
const { values } = parseArgs({
options: {
@@ -35,7 +36,7 @@ let outcome;
if (!values["non-interactive"]) {
finish({
kind: "failed",
- error: { message: "This setup requires --non-interactive." },
+ error: { message: "This deterministic setup requires --non-interactive." },
});
} else if (answers.phoneNumber === undefined) {
record("phone.requested");
@@ -52,7 +53,7 @@ if (!values["non-interactive"]) {
},
},
});
-} else if (!/^\+[1-9]\d{7,14}$/u.test(String(answers.phoneNumber))) {
+} else if (answers.phoneNumber !== EXPECTED_PHONE) {
record("phone.rejected", { supplied: String(answers.phoneNumber) });
finish({
kind: "blocked",
@@ -65,26 +66,25 @@ if (!values["non-interactive"]) {
required: true,
sensitive: false,
},
- issue: { code: "invalid_answer", message: "Use an E.164 phone number." },
+ issue: { code: "invalid_answer", message: "Use the user's supplied phone number." },
},
});
} else {
- const phoneNumber = String(answers.phoneNumber);
if (!state.projectCreated) {
state.projectCreated = true;
- record("project.created", { id: "photon-project" });
+ record("project.created", { id: "mock-imessage-project" });
}
if (!state.phoneRegistered) {
state.phoneRegistered = true;
- record("phone.registered", { phoneNumber });
+ record("phone.registered", { phoneNumber: EXPECTED_PHONE });
}
persist();
record("setup.completed");
finish({
kind: "completed",
facts: [
- { label: "Provider project", value: "photon-project" },
- { label: "Phone number", value: phoneNumber, kind: "phone" },
+ { label: "Provider project", value: "mock-imessage-project" },
+ { label: "Phone number", value: EXPECTED_PHONE, kind: "phone" },
],
deploymentRequired: true,
});
diff --git a/apps/benchmarks/lib/setups/mock-imessage-setup/package.json b/apps/benchmarks/lib/setups/mock-imessage-setup/package.json
index 683935e76..48d4da80c 100644
--- a/apps/benchmarks/lib/setups/mock-imessage-setup/package.json
+++ b/apps/benchmarks/lib/setups/mock-imessage-setup/package.json
@@ -1,9 +1,9 @@
{
- "name": "@photon-ai/setup",
+ "name": "@eve-internal/mock-imessage-setup",
"version": "0.0.0",
"private": true,
"bin": {
- "photon-setup": "./cli.mjs"
+ "mock-imessage-setup": "./cli.mjs"
},
"type": "module"
}
diff --git a/apps/benchmarks/package.json b/apps/benchmarks/package.json
index 6d9f8257e..f02db7cd1 100644
--- a/apps/benchmarks/package.json
+++ b/apps/benchmarks/package.json
@@ -5,7 +5,9 @@
"type": "module",
"scripts": {
"benchmark": "node run.mjs",
- "test": "node --test lib/*.test.mjs",
+ "benchmark:publish": "node publish.mjs",
+ "benchmark:export": "node scripts/export-results.mjs",
+ "test": "node --test lib/*.test.mjs scripts/*.test.mjs",
"typecheck": "tsc --noEmit"
},
"dependencies": {
@@ -13,7 +15,8 @@
"@ai-sdk/harness-codex": "1.0.69",
"@ai-sdk/sandbox-vercel": "1.0.67",
"@vercel/agent-eval": "1.5.0",
- "@vercel/sandbox": "catalog:"
+ "@vercel/sandbox": "catalog:",
+ "jiti": "2.7.0"
},
"devDependencies": {
"@types/node": "catalog:",
diff --git a/apps/benchmarks/publish.mjs b/apps/benchmarks/publish.mjs
new file mode 100644
index 000000000..c3aa817fc
--- /dev/null
+++ b/apps/benchmarks/publish.mjs
@@ -0,0 +1,200 @@
+#!/usr/bin/env node
+
+import { execFileSync, spawnSync } from "node:child_process";
+import { existsSync, mkdirSync, readdirSync, rmSync, writeFileSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { parseArgs } from "node:util";
+
+import {
+ authoringTreatments,
+ publishedBenchmark,
+ publishedExperimentId,
+} from "./lib/benchmark-config.ts";
+import { revisionSubject } from "./lib/source.mjs";
+
+const appRoot = dirname(fileURLToPath(import.meta.url));
+const repositoryRoot = resolve(appRoot, "../..");
+const evalsRoot = join(appRoot, "evals");
+const experimentsRoot = join(appRoot, "experiments");
+const { values } = parseArgs({
+ args: process.argv.slice(2),
+ options: {
+ "allow-dirty": { type: "boolean" },
+ dry: { type: "boolean" },
+ force: { type: "boolean" },
+ revision: { type: "string", default: "origin/main" },
+ help: { type: "boolean", short: "h" },
+ },
+ strict: true,
+});
+
+if (values.help) {
+ console.log(`Usage:
+ pnpm benchmark:publish [--revision origin/main] [--dry]
+ pnpm benchmark:publish [--revision <revision>] [--force] [--allow-dirty]`);
+ process.exit(0);
+}
+if (values.dry && values.force) throw new Error("--dry and --force cannot be combined.");
+if (!values.dry && !values["allow-dirty"] && !checkCleanWorkingTree()) process.exit(1);
+if (values["allow-dirty"]) {
+ console.warn(
+ "Warning: publishing from a dirty working tree is not reproducible from committed source.",
+ );
+}
+
+const revision = git(["rev-parse", "--verify", `${values.revision}^{commit}`]).trim();
+const subject = revisionSubject(repositoryRoot, revision, "published");
+const experimentNames = authoringTreatments.map(publishedExperimentId);
+
+prepareFixtures();
+writeExperiments(subject, revision);
+
+console.log(`> eve revision: ${revision}`);
+console.log(
+ `> model: ${publishedBenchmark.modelDisplayName} through ${publishedBenchmark.harness}`,
+);
+console.log(`> treatments: ${authoringTreatments.join(", ")}`);
+console.log(`> runs per cell: ${publishedBenchmark.runs}`);
+
+const executable = join(appRoot, "node_modules/.bin/agent-eval");
+if (values.dry) {
+ runAgentEval(["status", ...experimentNames]);
+ process.exit(0);
+}
+
+const cells = experimentNames.flatMap((experiment) =>
+ fixtureNames().map((caseId) => ({ experiment, caseId })),
+);
+for (const [index, cell] of cells.entries()) {
+ console.log(`\n> cell ${index + 1}/${cells.length}: ${cell.experiment} / ${cell.caseId}`);
+ runAgentEval(
+ ["run", cell.experiment, ...(values.force ? ["--force"] : [])],
+ { EVE_BENCHMARK_EVAL: cell.caseId },
+ false,
+ );
+}
+
+const pending = benchmarkStatus();
+if (pending.totalRun > 0) {
+ console.error(
+ `\nCannot publish: ${pending.totalRun} benchmark cell(s) still need a valid result.`,
+ );
+ console.error("Fix any infrastructure failures, then run pnpm benchmark:publish again.");
+ process.exit(1);
+}
+
+const exported = spawnSync(
+ process.execPath,
+ [join(appRoot, "scripts/export-results.mjs"), "--revision", revision],
+ { cwd: appRoot, stdio: "inherit", env: process.env },
+);
+if (exported.error) throw exported.error;
+process.exit(exported.status ?? 1);
+
+function runAgentEval(args, extraEnv = {}, exitOnFailure = true) {
+ const result = spawnSync(executable, args, {
+ cwd: appRoot,
+ stdio: "inherit",
+ env: { ...process.env, ...extraEnv },
+ });
+ if (result.error) throw result.error;
+ if (exitOnFailure && result.status !== 0) process.exit(result.status ?? 1);
+ return result.status ?? 1;
+}
+
+function benchmarkStatus() {
+ const result = spawnSync(executable, ["status", ...experimentNames, "--json"], {
+ cwd: appRoot,
+ encoding: "utf8",
+ env: process.env,
+ });
+ if (result.error) throw result.error;
+ if (result.status !== 0) {
+ process.stderr.write(result.stderr);
+ process.exit(result.status ?? 1);
+ }
+ return JSON.parse(result.stdout);
+}
+
+function checkCleanWorkingTree() {
+ const entries = git([
+ "status",
+ "--porcelain",
+ "--untracked-files=all",
+ "--",
+ ".",
+ ":!pnpm-lock.yaml",
+ ])
+ .trimEnd()
+ .split("\n")
+ .filter(Boolean);
+ if (entries.length === 0) return true;
+
+ const shown = entries.slice(0, 12);
+ console.error("Cannot publish canonical benchmark results from a dirty working tree.\n");
+ console.error("Blocking changes:");
+ for (const entry of shown) console.error(` ${entry}`);
+ if (entries.length > shown.length) {
+ console.error(` … and ${entries.length - shown.length} more`);
+ }
+ console.error(`
+Publishing records results against committed source so another run can reproduce them.
+Commit, stash, or remove these changes, then run:
+
+ pnpm benchmark:publish
+
+To bypass this check for a local run:
+
+ pnpm benchmark:publish --allow-dirty
+
+To inspect pending benchmark cells without publishing:
+
+ pnpm benchmark:publish --dry`);
+ return false;
+}
+
+function prepareFixtures() {
+ for (const name of fixtureNames()) {
+ const fixtureRoot = join(evalsRoot, name);
+ writeFileSync(join(fixtureRoot, "PROMPT.md"), "");
+ writeFileSync(
+ join(fixtureRoot, "package.json"),
+ `${JSON.stringify({ name: `eve-authoring-${name}`, private: true, type: "module" }, null, 2)}\n`,
+ );
+ }
+}
+
+function fixtureNames() {
+ return readdirSync(evalsRoot, { withFileTypes: true })
+ .filter((entry) => entry.isDirectory() && existsSync(join(evalsRoot, entry.name, "CASE.ts")))
+ .map((entry) => entry.name)
+ .sort();
+}
+
+function writeExperiments(subject, revision) {
+ rmSync(experimentsRoot, { recursive: true, force: true });
+ mkdirSync(experimentsRoot, { recursive: true });
+ const archiveName = `published-${revision.slice(0, 12)}.source.tar.gz`;
+ writeFileSync(join(experimentsRoot, archiveName), subject.archive);
+
+ for (const treatment of authoringTreatments) {
+ const experimentName = publishedExperimentId(treatment);
+ writeFileSync(
+ join(experimentsRoot, `${experimentName}.ts`),
+ `import { readFileSync } from "node:fs";\n` +
+ `import { authoringExperiment } from "../lib/experiment.js";\n\n` +
+ `export default authoringExperiment({\n` +
+ ` archive: readFileSync(new URL(${JSON.stringify(`./${archiveName}`)}, import.meta.url)),\n` +
+ ` digest: ${JSON.stringify(subject.digest)},\n` +
+ ` dependencyDigest: ${JSON.stringify(subject.dependencyDigest)},\n` +
+ ` runs: ${publishedBenchmark.runs},\n` +
+ ` treatment: ${JSON.stringify(treatment)},\n` +
+ `});\n`,
+ );
+ }
+}
+
+function git(args) {
+ return execFileSync("git", args, { cwd: repositoryRoot, encoding: "utf8" });
+}
diff --git a/apps/benchmarks/run.mjs b/apps/benchmarks/run.mjs
index 6860024b5..36ff3071a 100644
--- a/apps/benchmarks/run.mjs
+++ b/apps/benchmarks/run.mjs
@@ -6,6 +6,7 @@ import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { parseArgs } from "node:util";
+import { parseAuthoringTreatment } from "./lib/benchmark-config.ts";
import { revisionSubject, workingTreeSubject } from "./lib/source.mjs";
const appRoot = dirname(fileURLToPath(import.meta.url));
@@ -20,6 +21,7 @@ const { values, positionals } = parseArgs({
head: { type: "string" },
dry: { type: "boolean" },
runs: { type: "string" },
+ treatment: { type: "string", default: "guided" },
verbose: { type: "boolean" },
help: { type: "boolean", short: "h" },
},
@@ -36,6 +38,7 @@ if (values.head !== undefined && values.base === undefined) {
}
const runs = parseRuns(values.runs);
const selectedEval = positionals[0];
+const treatment = parseAuthoringTreatment(values.treatment);
if (values.verbose && (selectedEval === undefined || runs !== 1 || values.base !== undefined)) {
throw new Error("--verbose requires one eval, one run, and no revision comparison.");
}
@@ -56,7 +59,7 @@ const subjects =
prepareFixtures();
mkdirSync(join(appRoot, "results"), { recursive: true });
-writeExperiments(subjects, runs, values.verbose ?? false);
+writeExperiments(subjects, runs, treatment, values.verbose ?? false);
const executable = join(appRoot, "node_modules/.bin/agent-eval");
const experimentNames = subjects.map((subject) => subject.label);
const args = values.dry ? ["status", ...experimentNames] : ["run", ...experimentNames, "--force"];
@@ -88,7 +91,7 @@ function fixtureNames() {
.map((entry) => entry.name);
}
-function writeExperiments(subjects, runs, verbose) {
+function writeExperiments(subjects, runs, treatment, verbose) {
rmSync(experimentsRoot, { recursive: true, force: true });
mkdirSync(experimentsRoot, { recursive: true });
for (const subject of subjects) {
@@ -103,6 +106,7 @@ function writeExperiments(subjects, runs, verbose) {
` digest: ${JSON.stringify(subject.digest)},\n` +
` dependencyDigest: ${JSON.stringify(subject.dependencyDigest)},\n` +
` runs: ${runs},\n` +
+ ` treatment: ${JSON.stringify(treatment)},\n` +
` verbose: ${verbose},\n` +
`});\n`,
);
@@ -119,6 +123,6 @@ function parseRuns(value) {
function usage() {
console.log(`Usage:
- pnpm benchmark [eval-name] [--runs N] [--dry] [--verbose]
- pnpm benchmark [eval-name] --base <revision> [--head <revision>] [--runs N] [--dry]`);
+ pnpm benchmark [eval-name] [--runs N] [--treatment baseline|guided] [--dry] [--verbose]
+ pnpm benchmark [eval-name] --base <revision> [--head <revision>] [--runs N] [--treatment baseline|guided] [--dry]`);
}
diff --git a/apps/benchmarks/scripts/export-results.mjs b/apps/benchmarks/scripts/export-results.mjs
new file mode 100644
index 000000000..38de77550
--- /dev/null
+++ b/apps/benchmarks/scripts/export-results.mjs
@@ -0,0 +1,159 @@
+#!/usr/bin/env node
+
+import { execFileSync } from "node:child_process";
+import { createHash } from "node:crypto";
+import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { parseArgs } from "node:util";
+
+import {
+ authoringTreatments,
+ publishedBenchmark,
+ publishedExperimentId,
+} from "../lib/benchmark-config.ts";
+
+const appRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
+const repositoryRoot = resolve(appRoot, "../..");
+const evalsRoot = join(appRoot, "evals");
+const resultsRoot = join(appRoot, "results");
+const outputPath = join(repositoryRoot, "apps/docs/lib/evals/benchmark-results.json");
+const { values } = parseArgs({
+ args: process.argv.slice(2),
+ options: {
+ revision: { type: "string" },
+ output: { type: "string" },
+ help: { type: "boolean", short: "h" },
+ },
+ strict: true,
+});
+
+if (values.help) {
+ console.log("Usage: node scripts/export-results.mjs --revision <full-sha> [--output <path>]");
+ process.exit(0);
+}
+if (values.revision === undefined || !/^[0-9a-f]{40}$/u.test(values.revision)) {
+ throw new Error("--revision must be a full 40-character Git commit SHA.");
+}
+
+const caseIds = canonicalCaseIds();
+const experimentIds = authoringTreatments.map(publishedExperimentId);
+const stale = staleCells(experimentIds);
+const results = [];
+
+for (const treatment of authoringTreatments) {
+ const experimentId = publishedExperimentId(treatment);
+ for (const caseId of caseIds) {
+ const staleStatus = stale.get(experimentId);
+ const status = staleStatus?.changed.has(caseId)
+ ? "stale"
+ : staleStatus?.new.has(caseId)
+ ? "missing"
+ : undefined;
+ const measured = latestValidResult(experimentId, caseId);
+ if (status !== undefined || measured === undefined) {
+ results.push({ experimentId, caseId, status: status ?? "missing" });
+ continue;
+ }
+ results.push({ experimentId, caseId, status: "current", ...measured });
+ }
+}
+
+const output = {
+ schemaVersion: 1,
+ generatedAt: new Date().toISOString(),
+ suite: {
+ eveRevision: values.revision,
+ caseFingerprint: caseFingerprint(caseIds),
+ caseCount: caseIds.length,
+ runsPerCell: publishedBenchmark.runs,
+ },
+ experiments: authoringTreatments.map((treatment) => ({
+ id: publishedExperimentId(treatment),
+ groupId: publishedBenchmark.groupId,
+ model: publishedBenchmark.model,
+ modelDisplayName: publishedBenchmark.modelDisplayName,
+ harness: publishedBenchmark.harness,
+ treatment,
+ })),
+ results,
+};
+
+const destination =
+ values.output === undefined ? outputPath : resolve(process.cwd(), values.output);
+mkdirSync(dirname(destination), { recursive: true });
+writeFileSync(destination, `${JSON.stringify(output, null, 2)}\n`);
+console.log(`Exported ${results.length} benchmark cells to ${destination}`);
+
+function caseFingerprint(caseIds) {
+ const hash = createHash("sha256");
+ for (const caseId of caseIds) {
+ const caseRoot = join(evalsRoot, caseId);
+ for (const path of findFiles(caseRoot).sort()) {
+ const relativePath = path.slice(caseRoot.length + 1);
+ if (relativePath === "PROMPT.md" || relativePath === "package.json") continue;
+ hash.update(caseId).update("\0").update(relativePath).update("\0");
+ hash.update(readFileSync(path)).update("\0");
+ }
+ }
+ return hash.digest("hex");
+}
+
+function canonicalCaseIds() {
+ return readdirSync(evalsRoot, { withFileTypes: true })
+ .filter((entry) => entry.isDirectory() && existsSync(join(evalsRoot, entry.name, "CASE.ts")))
+ .map((entry) => entry.name)
+ .sort();
+}
+
+function staleCells(experimentIds) {
+ const executable = join(appRoot, "node_modules/.bin/agent-eval");
+ const raw = execFileSync(executable, ["status", ...experimentIds, "--json"], {
+ cwd: appRoot,
+ encoding: "utf8",
+ maxBuffer: 64 * 1024 * 1024,
+ });
+ const parsed = JSON.parse(raw);
+ const work = Array.isArray(parsed.work) ? parsed.work : [];
+ return new Map(
+ work.map((entry) => [
+ entry.experiment,
+ {
+ new: new Set(Array.isArray(entry.new) ? entry.new : []),
+ changed: new Set(Array.isArray(entry.changed) ? entry.changed : []),
+ },
+ ]),
+ );
+}
+
+function latestValidResult(experimentId, caseId) {
+ const experimentRoot = join(resultsRoot, experimentId);
+ if (!existsSync(experimentRoot)) return undefined;
+ const summaries = findFiles(experimentRoot, "summary.json")
+ .filter((path) => path.split("/").includes(caseId))
+ .sort((left, right) => statSync(right).mtimeMs - statSync(left).mtimeMs);
+
+ for (const summaryPath of summaries) {
+ const summary = JSON.parse(readFileSync(summaryPath, "utf8"));
+ if (summary.valid === false) continue;
+ if (!Number.isFinite(summary.totalRuns) || summary.totalRuns < 1) continue;
+ if (!Number.isFinite(summary.passedRuns) || !Number.isFinite(summary.meanDuration)) continue;
+ return {
+ passedRuns: summary.passedRuns,
+ validRuns: summary.totalRuns,
+ meanDurationMs: Math.round(summary.meanDuration * 1000),
+ measuredAt: statSync(summaryPath).mtime.toISOString(),
+ };
+ }
+ return undefined;
+}
+
+function findFiles(root, fileName) {
+ const matches = [];
+ for (const entry of readdirSync(root, { withFileTypes: true })) {
+ const path = join(root, entry.name);
+ if (entry.isDirectory()) matches.push(...findFiles(path, fileName));
+ else if (fileName === undefined || entry.name === fileName) matches.push(path);
+ }
+ return matches;
+}
diff --git a/apps/benchmarks/scripts/export-results.test.mjs b/apps/benchmarks/scripts/export-results.test.mjs
new file mode 100644
index 000000000..76e263a25
--- /dev/null
+++ b/apps/benchmarks/scripts/export-results.test.mjs
@@ -0,0 +1,45 @@
+import assert from "node:assert/strict";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { test } from "node:test";
+import { execFileSync } from "node:child_process";
+
+const appRoot = new URL("..", import.meta.url);
+
+test("exports missing cells without publishing private artifacts", () => {
+ const directory = mkdtempSync(join(tmpdir(), "eve-benchmark-export-"));
+ const outputPath = join(directory, "results.json");
+ const resultsPath = new URL("../results", import.meta.url).pathname;
+ const savedResultsPath = join(directory, "saved-results");
+ try {
+ if (existsSync(resultsPath)) renameSync(resultsPath, savedResultsPath);
+ mkdirSync(resultsPath, { recursive: true });
+ execFileSync(
+ process.execPath,
+ [
+ new URL("./export-results.mjs", import.meta.url).pathname,
+ "--revision",
+ "a".repeat(40),
+ "--output",
+ outputPath,
+ ],
+ { cwd: appRoot, stdio: "pipe" },
+ );
+ const raw = readFileSync(outputPath, "utf8");
+ const output = JSON.parse(raw);
+
+ assert.equal(output.schemaVersion, 1);
+ assert.equal(output.suite.caseCount, 3);
+ assert.match(output.suite.caseFingerprint, /^[0-9a-f]{64}$/u);
+ assert.equal(output.results.length, 6);
+ assert.ok(output.results.every((result) => result.status === "missing"));
+ for (const privateField of ["transcript", "commands", "worldEvents", "files"]) {
+ assert.equal(raw.includes(`\"${privateField}\"`), false);
+ }
+ } finally {
+ rmSync(resultsPath, { recursive: true, force: true });
+ if (existsSync(savedResultsPath)) renameSync(savedResultsPath, resultsPath);
+ rmSync(directory, { recursive: true, force: true });
+ }
+});
diff --git a/package.json b/package.json
index d76d18c49..c7ec46db7 100644
--- a/package.json
+++ b/package.json
@@ -13,6 +13,7 @@
"type": "module",
"scripts": {
"benchmark": "pnpm --filter @eve-internal/benchmarks benchmark",
+ "benchmark:publish": "node apps/benchmarks/publish.mjs",
"build": "turbo run build --filter='./packages/*'",
"changeset": "changeset",
"check:deps": "syncpack lint",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index a3fde8db2..922745dff 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -146,6 +146,9 @@ importers:
'@vercel/sandbox':
specifier: 'catalog:'
version: 2.8.0
+ jiti:
+ specifier: 2.7.0
+ version: 2.7.0
devDependencies:
'@types/node':
specifier: 'catalog:'