The Open Framework for Building Agents
feat: publish authoring benchmark results (#2057)
Colton Padden4f78a2d336a7
diff --git a/.gitignore b/.gitignoreindex f06caebc0..305d74c08 100644--- a/.gitignore+++ b/.gitignore@@ -51,6 +51,7 @@ Thumbs.db# IDE/editor.codex/+.githooks/.vscode/.idea/diff --git a/apps/benchmarks/README.md b/apps/benchmarks/README.mdindex 752377095..54e30db5a 100644--- a/apps/benchmarks/README.md+++ b/apps/benchmarks/README.md@@ -13,6 +13,7 @@ Git does not ignore:pnpm benchmark author-000-imessagepnpm benchmarkpnpm benchmark author-000-imessage --runs 3+pnpm benchmark author-000-imessage --treatment baselinepnpm benchmark author-000-imessage --drypnpm benchmark author-000-imessage --verbose```@@ -37,10 +38,35 @@ commits do not need to be pushed. Dependency downloads are cached by lockfile, schanges reuse the prepared pnpm store. For one eval and one run, `--verbose` streams setupphases, assistant text, tool calls, grading, and build progress.+Local runs use the `guided` treatment by default, which keeps the `AGENTS.md` and aliases generated+by `eve init`. Pass `--treatment baseline` to remove those files before the coding agent starts.+Results are written under `apps/benchmarks/results/`. Each run includes the transcript,grader output, summary, and copied project files. Vercel Sandbox and AI Gateway credentials arerequired.+## Publish canonical results++Canonical publication compares the `baseline` and `guided` treatments with the same eve revision,+model, harness, cases, and graders. It requires a clean working tree and defaults to `origin/main`:++```sh+pnpm benchmark:publish --dry+pnpm benchmark:publish+pnpm benchmark:publish --revision <commit>+```++Pass `--allow-dirty` only for a local, noncanonical run. It bypasses the clean-tree check, so its+results cannot be reproduced from committed source and should not be committed as published data.++Each cell runs three times. Completed cells are memoized by `@vercel/agent-eval`; pass `--force` only+when every cell should run again. A successful run writes aggregate results to+`apps/docs/lib/evals/benchmark-results.json`. The public file contains outcomes and timing, not+transcripts, generated files, command logs, or synthetic world events.++Changed and newly added cases export as unavailable until they run. The exporter does not carry an+older measurement forward as current.+## Add a caseCreate the next `evals/author-NNN-*` directory with:diff --git a/apps/benchmarks/evals/author-000-imessage/CASE.ts b/apps/benchmarks/evals/author-000-imessage/CASE.tsindex 278d939a9..106b32061 100644--- a/apps/benchmarks/evals/author-000-imessage/CASE.ts+++ b/apps/benchmarks/evals/author-000-imessage/CASE.ts@@ -1,18 +1,14 @@import { defineAuthoringCase, simpleProject } from "../../lib/authoring-case.js";import { imessageSetup } from "../../lib/setups/imessage.js";-const PHONE_NUMBER = "+447700900123";-const PHONE_NUMBER_QUESTION =- /(?:what|which|provide|share|send|need)[^.!?\n]{0,100}(?:phone|imessage)[^.!?\n]{0,50}number|(?:phone|imessage)[^.!?\n]{0,50}number[^.!?\n]{0,100}(?:use|receive|register|provide|share|send|need)|(?:blocked|waiting)[^.!?\n]{0,120}(?:phone|imessage)[^.!?\n]{0,50}number/i;+const PHONE_NUMBER = "+15551234567";export default defineAuthoringCase({startingPoint: simpleProject,setup: imessageSetup,async interact({ send }) {- const firstTurn = await send(- "Let me talk to this agent via iMessage. Configure it locally, but do not deploy it.",- );- if (!PHONE_NUMBER_QUESTION.test(firstTurn.text)) {+ const firstTurn = await send("Let me talk to this agent via iMessage.");+ if (!/phone number|imessage number|number should/i.test(firstTurn.text)) {throw new Error(`Expected the agent to ask for the user's phone number on its first turn. Received: ${JSON.stringify(firstTurn.text)}`,);diff --git a/apps/benchmarks/evals/author-000-imessage/EVAL.ts b/apps/benchmarks/evals/author-000-imessage/EVAL.tsindex 5fef3c4be..5a16d2416 100644--- a/apps/benchmarks/evals/author-000-imessage/EVAL.ts+++ b/apps/benchmarks/evals/author-000-imessage/EVAL.ts@@ -1,9 +1,8 @@import { existsSync, readFileSync } from "node:fs";-import { join } from "node:path";import { expect, test } from "vitest";-import { authoringEval, workspace } from "./grader.js";+import { authoringEval } from "./grader.js";const { commands, transcript, worldEvents } = authoringEval();const commandLog = commands.join("\n");@@ -15,7 +14,7 @@ test("installs the discovered iMessage registry item through the headless setuptest("asks for and uses the phone number from the follow-up turn", () => {expect(transcript.some((entry) => /phone number/i.test(entry.content))).toBe(true);- expect(transcript.some((entry) => entry.content.includes("+447700900123"))).toBe(true);+ expect(transcript.some((entry) => entry.content.includes("+15551234567"))).toBe(true);});test("completes the synthetic provider setup decision tree", () => {@@ -23,11 +22,11 @@ test("completes the synthetic provider setup decision tree", () => {expect.arrayContaining(["project.created", "phone.registered", "setup.completed"]),);const registration = worldEvents.find((event) => event.type === "phone.registered");- expect(registration?.data?.phoneNumber).toBe("+447700900123");+ expect(registration?.data?.phoneNumber).toBe("+15551234567");});test("creates an iMessage channel and leaves the project valid", () => {- const channelPath = join(workspace, "agent/channels/imessage.ts");+ const channelPath = "agent/channels/imessage.ts";expect(existsSync(channelPath)).toBe(true);expect(readFileSync(channelPath, "utf8")).toContain("photonIMessageChannel");});diff --git a/apps/benchmarks/evals/author-001-weather-tool/EVAL.ts b/apps/benchmarks/evals/author-001-weather-tool/EVAL.tsindex 41703f74d..e268d7163 100644--- a/apps/benchmarks/evals/author-001-weather-tool/EVAL.ts+++ b/apps/benchmarks/evals/author-001-weather-tool/EVAL.ts@@ -1,11 +1,8 @@import { existsSync, readFileSync } from "node:fs";-import { join } from "node:path";import { expect, test } from "vitest";-import { workspace } from "./grader.js";--const toolPath = join(workspace, "agent/tools/get_weather.ts");+const toolPath = "agent/tools/get_weather.ts";test("creates a filesystem-named weather tool", () => {expect(existsSync(toolPath)).toBe(true);diff --git a/apps/benchmarks/evals/author-002-new-project/CASE.ts b/apps/benchmarks/evals/author-002-new-project/CASE.tsindex b6e64824e..0418635f0 100644--- a/apps/benchmarks/evals/author-002-new-project/CASE.ts+++ b/apps/benchmarks/evals/author-002-new-project/CASE.ts@@ -4,7 +4,7 @@ export default defineAuthoringCase({startingPoint: emptyProject,async interact({ send }) {await send(- "Create a new eve project in this directory for a concise travel assistant called Wayfinder. It should use the default model and be ready to build.",+ "Create a new eve project directly in the current working directory for a concise travel assistant called Wayfinder. Do not create a subdirectory: initialize the current directory (for example, with `eve init .`). It should use the default model and be ready to build.",);},});diff --git a/apps/benchmarks/evals/author-002-new-project/EVAL.ts b/apps/benchmarks/evals/author-002-new-project/EVAL.tsindex b54c239cc..cc5d10799 100644--- a/apps/benchmarks/evals/author-002-new-project/EVAL.ts+++ b/apps/benchmarks/evals/author-002-new-project/EVAL.ts@@ -1,18 +1,15 @@import { existsSync, readFileSync } from "node:fs";-import { join } from "node:path";import { expect, test } from "vitest";-import { subjectDefaultAgentModel, workspace } from "./grader.js";--const projectPath = (path: string) => join(workspace, path);+import { subjectDefaultAgentModel } from "./grader.js";test("creates a complete eve project in place", () => {- expect(existsSync(projectPath("agent/agent.ts"))).toBe(true);- expect(existsSync(projectPath("agent/channels/eve.ts"))).toBe(true);- expect(existsSync(projectPath("agent/instructions.md"))).toBe(true);+ expect(existsSync("agent/agent.ts")).toBe(true);+ expect(existsSync("agent/channels/eve.ts")).toBe(true);+ expect(existsSync("agent/instructions.md")).toBe(true);- const packageJson = JSON.parse(readFileSync(projectPath("package.json"), "utf8")) as {+ const packageJson = JSON.parse(readFileSync("package.json", "utf8")) as {dependencies?: Record<string, string>;scripts?: Record<string, string>;};@@ -21,9 +18,9 @@ test("creates a complete eve project in place", () => {});test("authors the requested identity without replacing the default model", () => {- expect(readFileSync(projectPath("agent/instructions.md"), "utf8")).toMatch(/Wayfinder/i);- expect(readFileSync(projectPath("agent/instructions.md"), "utf8")).toMatch(/travel/i);- expect(readFileSync(projectPath("agent/agent.ts"), "utf8")).toContain(+ expect(readFileSync("agent/instructions.md", "utf8")).toMatch(/Wayfinder/i);+ expect(readFileSync("agent/instructions.md", "utf8")).toMatch(/travel/i);+ expect(readFileSync("agent/agent.ts", "utf8")).toContain(`model: "${subjectDefaultAgentModel()}"`,);});diff --git a/apps/benchmarks/lib/authoring-case.ts b/apps/benchmarks/lib/authoring-case.tsindex 0a32d0171..bd9481e85 100644--- a/apps/benchmarks/lib/authoring-case.ts+++ b/apps/benchmarks/lib/authoring-case.ts@@ -30,7 +30,7 @@ export interface AuthoringSetup {export interface AuthoringTurn {readonly text: string;- readonly toolCalls: ReadonlyArray<{ input: unknown }>;+ readonly toolCalls: ReadonlyArray<{ name: string; input: unknown }>;}export interface AuthoringInteractionContext {@@ -46,12 +46,12 @@ export interface AuthoringCase {}export const emptyProject: AuthoringStartingPoint = {- id: "empty",+ id: "empty-v2",workspace: "empty",};export const simpleProject: AuthoringStartingPoint = {- id: "simple",+ id: "simple-v2",workspace: "scaffolded",};diff --git a/apps/benchmarks/lib/benchmark-config.ts b/apps/benchmarks/lib/benchmark-config.tsnew file mode 100644index 000000000..7d62ada2a--- /dev/null+++ b/apps/benchmarks/lib/benchmark-config.ts@@ -0,0 +1,26 @@+export const AUTHORING_MODEL = "claude-sonnet-4-6";++export const authoringTreatments = ["baseline", "guided"] as const;++export type AuthoringTreatment = (typeof authoringTreatments)[number];++export const publishedBenchmark = {+ groupId: "claude-sonnet-4-6-claude-code",+ model: AUTHORING_MODEL,+ modelDisplayName: "Claude Sonnet 4.6",+ harness: "Claude Code",+ runs: 3,+} as const;++export function publishedExperimentId(treatment: AuthoringTreatment): string {+ return `${publishedBenchmark.groupId}--${treatment}`;+}++export function parseAuthoringTreatment(value: string): AuthoringTreatment {+ if (authoringTreatments.includes(value as AuthoringTreatment)) {+ return value as AuthoringTreatment;+ }+ throw new Error(+ `Unknown treatment ${JSON.stringify(value)}. Expected one of: ${authoringTreatments.join(", ")}.`,+ );+}diff --git a/apps/benchmarks/lib/dependency-sandbox.ts b/apps/benchmarks/lib/dependency-sandbox.tsindex dec75c7d9..b2b9da024 100644--- a/apps/benchmarks/lib/dependency-sandbox.ts+++ b/apps/benchmarks/lib/dependency-sandbox.ts@@ -4,8 +4,7 @@ import { Sandbox } from "@vercel/sandbox";import { SOURCE_ARCHIVE_PATH, SOURCE_ROOT } from "./paths.js";-const dependencySnapshots = new Map<string, Promise<string>>();-const subjectSnapshots = new Map<string, Promise<string>>();+const snapshots = new Map<string, Promise<string>>();export function createDependencyCachedSandbox(options: {readonly archive: Uint8Array;@@ -14,53 +13,35 @@ export function createDependencyCachedSandbox(options: {readonly env: Readonly<Record<string, string>>;readonly log: (message: string) => void;}): HarnessV1SandboxProvider {- const sessionProvider = (snapshotId?: string) =>- snapshotId === undefined- ? createVercelSandbox({- runtime: "node24",- ports: [...options.ports],- timeout: 15 * 60_000,- env: { ...options.env },- networkPolicy: "allow-all",- })- : createVercelSandbox({- source: { type: "snapshot", snapshotId },- ports: [...options.ports],- timeout: 15 * 60_000,- env: { ...options.env },- networkPolicy: "allow-all",- });-+ let provider: Promise<HarnessV1SandboxProvider> | undefined;+ const resolveProvider = () => {+ options.log("[setup] preparing dependency cache");+ return (provider ??= dependencySnapshot(+ options.archive,+ options.dependencyDigest,+ options.log,+ ).then((snapshotId) =>+ createVercelSandbox({+ source: { type: "snapshot", snapshotId },+ ports: [...options.ports],+ timeout: 15 * 60_000,+ env: { ...options.env },+ networkPolicy: "allow-all",+ }),+ ));+ };return {specificationVersion: "harness-sandbox-v1",providerId: "eve-benchmark-vercel",- async createSession(request = {}) {- if (request.identity === undefined || request.onFirstCreate === undefined) {- return sessionProvider().createSession(request);- }- options.log("[setup] preparing dependency cache");- const dependencySnapshotId = await dependencySnapshot(- options.archive,- options.dependencyDigest,- options.log,- );- const subjectSnapshotId = await subjectSnapshot(- dependencySnapshotId,- request.identity,- options.env,- request.onFirstCreate,- request.abortSignal,- );- return sessionProvider(subjectSnapshotId).createSession({- abortSignal: request.abortSignal,- });+ async createSession(options) {+ return (await resolveProvider()).createSession(options);},- async resumeSession(request) {- const provider = sessionProvider();- if (provider.resumeSession === undefined) {+ async resumeSession(options) {+ const resolved = await resolveProvider();+ if (resolved.resumeSession === undefined) {throw new Error("Vercel Sandbox does not support session resume.");}- return provider.resumeSession(request);+ return resolved.resumeSession(options);},};}@@ -70,11 +51,11 @@ function dependencySnapshot(digest: string,log: (message: string) => void,): Promise<string> {- const name = `eve-benchmark-dependencies-${digest.slice(0, 24)}`;- let snapshot = dependencySnapshots.get(name);+ const name = `eve-benchmark-dependencies-v4-${digest.slice(0, 24)}`;+ let snapshot = snapshots.get(name);if (snapshot !== undefined) return snapshot;snapshot = createDependencySnapshot(name, archive, log);- dependencySnapshots.set(name, snapshot);+ snapshots.set(name, snapshot);return snapshot;}@@ -83,7 +64,6 @@ async function createDependencySnapshot(archive: Uint8Array,log: (message: string) => void,): Promise<string> {- let created = false;const sandbox = await Sandbox.getOrCreate({name,runtime: "node24",@@ -91,11 +71,10 @@ async function createDependencySnapshot(persistent: true,snapshotExpiration: 0,networkPolicy: "allow-all",- async onCreate(current) {- created = true;+ async onCreate(created) {log("[setup] fetching workspace dependencies");- await current.writeFiles([{ path: SOURCE_ARCHIVE_PATH, content: archive }]);- const command = await current.runCommand("bash", [+ await created.writeFiles([{ path: SOURCE_ARCHIVE_PATH, content: archive }]);+ const command = await created.runCommand("bash", ["-lc",`mkdir -p ${SOURCE_ROOT} && tar -xzf ${SOURCE_ARCHIVE_PATH} -C ${SOURCE_ROOT} && npm install --global [email protected] [email protected] && cd ${SOURCE_ROOT} && pnpm fetch --frozen-lockfile`,]);@@ -106,60 +85,8 @@ async function createDependencySnapshot(}},});- if (!created && sandbox.currentSnapshotId !== undefined) return sandbox.currentSnapshotId;- return stopWithSnapshot(sandbox, "Dependency");-}--function subjectSnapshot(- dependencySnapshotId: string,- identity: string,- env: Readonly<Record<string, string>>,- bootstrap: NonNullable<Parameters<HarnessV1SandboxProvider["createSession"]>[0]>["onFirstCreate"],- abortSignal?: AbortSignal,-): Promise<string> {- const name = `eve-benchmark-subject-${identity}`;- let snapshot = subjectSnapshots.get(name);- if (snapshot !== undefined) return snapshot;- snapshot = createSubjectSnapshot(name, dependencySnapshotId, env, bootstrap, abortSignal);- subjectSnapshots.set(name, snapshot);- return snapshot;-}--async function createSubjectSnapshot(- name: string,- dependencySnapshotId: string,- env: Readonly<Record<string, string>>,- bootstrap: NonNullable<Parameters<HarnessV1SandboxProvider["createSession"]>[0]>["onFirstCreate"],- abortSignal?: AbortSignal,-): Promise<string> {- let created = false;- const sandbox = await Sandbox.getOrCreate({- name,- source: { type: "snapshot", snapshotId: dependencySnapshotId },- timeout: 15 * 60_000,- env: { ...env },- persistent: true,- snapshotExpiration: 0,- networkPolicy: "allow-all",- signal: abortSignal,- async onCreate(current) {- created = true;- const provider = createVercelSandbox({ sandbox: current });- const session = await provider.createSession({ abortSignal });- await bootstrap!(session.restricted(), { abortSignal });- },- });- if (!created && sandbox.currentSnapshotId !== undefined) return sandbox.currentSnapshotId;- return stopWithSnapshot(sandbox, "Subject", abortSignal);-}--async function stopWithSnapshot(- sandbox: Sandbox,- label: string,- signal?: AbortSignal,-): Promise<string> {- const stopped = await sandbox.stop(signal === undefined ? undefined : { signal });- const snapshotId = stopped.snapshot?.id ?? sandbox.currentSnapshotId;- if (snapshotId === undefined) throw new Error(`${label} snapshot was not published.`);- return snapshotId;+ if (sandbox.currentSnapshotId !== undefined) return sandbox.currentSnapshotId;+ const stopped = await sandbox.stop();+ if (stopped.snapshot?.id === undefined) throw new Error("Dependency snapshot was not published.");+ return stopped.snapshot.id;}diff --git a/apps/benchmarks/lib/experiment.ts b/apps/benchmarks/lib/experiment.tsindex 2cf94eabb..d00b0cf6e 100644--- a/apps/benchmarks/lib/experiment.ts+++ b/apps/benchmarks/lib/experiment.ts@@ -1,18 +1,20 @@import type { ExperimentConfig } from "@vercel/agent-eval";import { registerAgent } from "@vercel/agent-eval";+import type { AuthoringTreatment } from "./benchmark-config.js";+import { AUTHORING_MODEL } from "./benchmark-config.js";import { createAuthoringAgent } from "./harness-agent.js";-import { AUTHORING_MODEL } from "./paths.js";export function authoringExperiment(options: {readonly archive: Uint8Array;readonly digest: string;readonly dependencyDigest: string;readonly runs?: number;+ readonly treatment: AuthoringTreatment;readonly verbose?: boolean;}): ExperimentConfig {const agent = createAuthoringAgent({- name: `eve-authoring-harness-${options.digest.slice(0, 12)}`,+ name: `eve-authoring-harness-${options.digest.slice(0, 12)}-${options.treatment}`,archive: options.archive,digest: options.digest,dependencyDigest: options.dependencyDigest,@@ -24,10 +26,13 @@ export function authoringExperiment(options: {evals: process.env.EVE_BENCHMARK_EVAL ?? "*",scripts: ["typecheck", "build"],runs: options.runs ?? 1,- earlyExit: options.runs === undefined,+ earlyExit: false,timeout: 900,sandbox: "vercel",copyFiles: "changed",- agentOptions: { agentsMd: true, verbose: options.verbose ?? false },+ agentOptions: {+ agentsMd: options.treatment === "guided",+ verbose: options.verbose ?? false,+ },};}diff --git a/apps/benchmarks/lib/grader.ts b/apps/benchmarks/lib/grader.tsindex 0bfa2474e..ecdd0defb 100644--- a/apps/benchmarks/lib/grader.ts+++ b/apps/benchmarks/lib/grader.ts@@ -1,6 +1,6 @@import { existsSync, readFileSync } from "node:fs";-import { AGENT_EVAL_DIRECTORY, SOURCE_ROOT, WORKSPACE_ENV, WORLD_EVENTS_PATH } from "./paths.js";+import { AGENT_EVAL_DIRECTORY, SOURCE_ROOT, WORLD_EVENTS_PATH } from "./paths.js";import type { AuthoringTranscriptEntry, AuthoringWorldEvent } from "./protocol.js";interface AgentEvalResults {@@ -13,9 +13,6 @@ interface AgentEvalResults {};}-export const workspace = process.env[WORKSPACE_ENV];-if (workspace === undefined) throw new Error(`${WORKSPACE_ENV} is required.`);-export interface AuthoringEvalResult {readonly commands: ReadonlyArray<string>;readonly transcript: ReadonlyArray<AuthoringTranscriptEntry>;diff --git a/apps/benchmarks/lib/harness-agent.ts b/apps/benchmarks/lib/harness-agent.tsindex bb5f0b07e..1737d31e0 100644--- a/apps/benchmarks/lib/harness-agent.ts+++ b/apps/benchmarks/lib/harness-agent.ts@@ -310,12 +310,21 @@ async function streamTurn(if (verbose && lineOpen) process.stdout.write("\n");return {text: await result.text,- toolCalls: await result.toolCalls,+ toolCalls: authoringToolCalls(await result.toolCalls),elapsedMs: performance.now() - startedAt,steps,};}+function authoringToolCalls(+ toolCalls: ReadonlyArray<{ name?: string; toolName?: string; input: unknown }>,+): ReadonlyArray<{ name: string; input: unknown }> {+ return toolCalls.flatMap((call) => {+ const name = call.toolName ?? call.name;+ return name === undefined ? [] : [{ name, input: call.input }];+ });+}+function formatToolCall(name: string, input: unknown): string {if (typeof input === "object" && input !== null) {const command = (input as { command?: unknown }).command;diff --git a/apps/benchmarks/lib/load-authoring-case.test.mjs b/apps/benchmarks/lib/load-authoring-case.test.mjsnew file mode 100644index 000000000..05699f080--- /dev/null+++ b/apps/benchmarks/lib/load-authoring-case.test.mjs@@ -0,0 +1,22 @@+import assert from "node:assert/strict";+import { readdir } from "node:fs/promises";+import { dirname, resolve } from "node:path";+import test from "node:test";+import { fileURLToPath } from "node:url";++import { createJiti } from "jiti";++const libRoot = dirname(fileURLToPath(import.meta.url));+const evalsRoot = resolve(libRoot, "../evals");+const jiti = createJiti(import.meta.url, { interopDefault: true, moduleCache: false });+const { loadAuthoringCase } = await jiti.import(resolve(libRoot, "load-authoring-case.ts"));++for (const entry of await readdir(evalsRoot, { withFileTypes: true })) {+ if (!entry.isDirectory()) continue;++ test(`loads ${entry.name}/CASE.ts`, async () => {+ const authoringCase = await loadAuthoringCase(resolve(evalsRoot, entry.name));+ assert.equal(typeof authoringCase.interact, "function");+ assert.ok(authoringCase.startingPoint);+ });+}diff --git a/apps/benchmarks/lib/load-authoring-case.ts b/apps/benchmarks/lib/load-authoring-case.tsnew file mode 100644index 000000000..a652953e4--- /dev/null+++ b/apps/benchmarks/lib/load-authoring-case.ts@@ -0,0 +1,23 @@+import { createJiti } from "jiti";++import type { AuthoringCase } from "./authoring-case.js";++export async function loadAuthoringCase(fixturePath: string): Promise<AuthoringCase> {+ const jiti = createJiti(import.meta.url, { interopDefault: true, moduleCache: false });+ let loaded: unknown = await jiti.import(`${fixturePath}/CASE.ts`);+ while (!isAuthoringCase(loaded) && hasDefaultExport(loaded)) loaded = loaded.default;+ if (!isAuthoringCase(loaded)) {+ throw new Error(`${fixturePath}/CASE.ts must export an authoring case as default.`);+ }+ return loaded;+}++function isAuthoringCase(value: unknown): value is AuthoringCase {+ if (typeof value !== "object" || value === null) return false;+ const candidate = value as Partial<AuthoringCase>;+ return candidate.startingPoint !== undefined && typeof candidate.interact === "function";+}++function hasDefaultExport(value: unknown): value is { default: unknown } {+ return typeof value === "object" && value !== null && "default" in value;+}diff --git a/apps/benchmarks/lib/protocol.ts b/apps/benchmarks/lib/protocol.tsindex a2762fb46..8c87ee4ae 100644--- a/apps/benchmarks/lib/protocol.ts+++ b/apps/benchmarks/lib/protocol.ts@@ -1,6 +1,10 @@export interface AuthoringTranscriptEntry {readonly role: "user" | "assistant";readonly content: string;+ readonly toolCalls?: ReadonlyArray<{+ readonly name: string;+ readonly input: unknown;+ }>;}export interface AuthoringWorldEvent {diff --git a/apps/benchmarks/lib/setups/imessage.ts b/apps/benchmarks/lib/setups/imessage.tsindex 992c2a882..91a5f7a49 100644--- a/apps/benchmarks/lib/setups/imessage.ts+++ b/apps/benchmarks/lib/setups/imessage.ts@@ -6,11 +6,11 @@ const REGISTRY_PORT = 4173;const REGISTRY_URL = `http://127.0.0.1:${REGISTRY_PORT}`;export const imessageSetup: AuthoringSetup = {- id: "imessage",+ id: "imessage-v1",ports: [REGISTRY_PORT],environment: { EVE_DEV_OFFICIAL_REGISTRY_URL: REGISTRY_URL },async onBootstrap({ run, artifactsRoot, write }) {- const setupRoot = `${artifactsRoot}/photon-setup`;+ const setupRoot = `${artifactsRoot}/mock-imessage-setup`;await Promise.all(["authoring-world.mjs", "cli.mjs", "package.json"].map((file) =>write(@@ -19,29 +19,29 @@ export const imessageSetup: AuthoringSetup = {),),);- await run(`pnpm add ${setupRoot}`);+ await run(`pnpm add ./${setupRoot}`);await run(`mkdir -p ${artifactsRoot}/registry/channel`);const channel = {$schema: "https://ui.shadcn.com/schema/registry-item.json",name: "channel/photon-imessage",title: "Photon iMessage",- description: "Connect an eve agent to iMessage through Photon.",+ description: "Connect an eve agent to iMessage through a deterministic provider setup flow.",files: [{path: "registry/channels/photon-imessage.ts",type: "registry:file",target: "agent/channels/imessage.ts",content:- 'import { photonIMessageChannel } from "eve/channels/photon";\n\nexport default photonIMessageChannel({ credentials: async () => ({ projectId: "photon-project", projectSecret: "photon-project-secret" }) });\n',+ 'import { photonIMessageChannel } from "eve/channels/photon";\n\nexport default photonIMessageChannel({ credentials: async () => ({ projectId: "mock-imessage-project", projectSecret: "mock-imessage-secret" }) });\n',},],meta: {eve: {setup: {- command: "photon-setup",- package: "@photon-ai/setup",- bin: "photon-setup",+ command: "mock-imessage-setup",+ package: "@eve-internal/mock-imessage-setup",+ bin: "mock-imessage-setup",args: [],},},@@ -50,7 +50,7 @@ export const imessageSetup: AuthoringSetup = {};const registry = {$schema: "https://ui.shadcn.com/schema/registry.json",- name: "eve-official",+ name: "eve-authoring-eval",homepage: REGISTRY_URL,items: [{@@ -68,9 +68,13 @@ export const imessageSetup: AuthoringSetup = {write(`${artifactsRoot}/registry/channel/photon-imessage.json`, JSON.stringify(channel)),]);},- async onSession({ run, artifactsRoot }) {+ async onSession({ run, artifactsRoot, write }) {await run(`nohup python3 -m http.server ${REGISTRY_PORT} --directory ${artifactsRoot}/registry >${artifactsRoot}/registry.log 2>&1 </dev/null &`,);+ await write(+ ".claude/settings.json",+ JSON.stringify({ env: { EVE_DEV_OFFICIAL_REGISTRY_URL: REGISTRY_URL } }),+ );},};diff --git a/apps/benchmarks/lib/setups/mock-imessage-setup/cli.mjs b/apps/benchmarks/lib/setups/mock-imessage-setup/cli.mjsindex 4e0894944..c95d6bcb4 100644--- a/apps/benchmarks/lib/setups/mock-imessage-setup/cli.mjs+++ b/apps/benchmarks/lib/setups/mock-imessage-setup/cli.mjs@@ -6,7 +6,8 @@ import { parseArgs } from "node:util";import { authoringStatePath, recordAuthoringEvent } from "./authoring-world.mjs";const PROTOCOL_VERSION = 2;-const STATE_FILE = authoringStatePath("photon-state");+const STATE_FILE = authoringStatePath("mock-imessage-state");+const EXPECTED_PHONE = process.env.EVE_AUTHORING_PHONE_NUMBER ?? "+15551234567";const { values } = parseArgs({options: {@@ -35,7 +36,7 @@ let outcome;if (!values["non-interactive"]) {finish({kind: "failed",- error: { message: "This setup requires --non-interactive." },+ error: { message: "This deterministic setup requires --non-interactive." },});} else if (answers.phoneNumber === undefined) {record("phone.requested");@@ -52,7 +53,7 @@ if (!values["non-interactive"]) {},},});-} else if (!/^\+[1-9]\d{7,14}$/u.test(String(answers.phoneNumber))) {+} else if (answers.phoneNumber !== EXPECTED_PHONE) {record("phone.rejected", { supplied: String(answers.phoneNumber) });finish({kind: "blocked",@@ -65,26 +66,25 @@ if (!values["non-interactive"]) {required: true,sensitive: false,},- issue: { code: "invalid_answer", message: "Use an E.164 phone number." },+ issue: { code: "invalid_answer", message: "Use the user's supplied phone number." },},});} else {- const phoneNumber = String(answers.phoneNumber);if (!state.projectCreated) {state.projectCreated = true;- record("project.created", { id: "photon-project" });+ record("project.created", { id: "mock-imessage-project" });}if (!state.phoneRegistered) {state.phoneRegistered = true;- record("phone.registered", { phoneNumber });+ record("phone.registered", { phoneNumber: EXPECTED_PHONE });}persist();record("setup.completed");finish({kind: "completed",facts: [- { label: "Provider project", value: "photon-project" },- { label: "Phone number", value: phoneNumber, kind: "phone" },+ { label: "Provider project", value: "mock-imessage-project" },+ { label: "Phone number", value: EXPECTED_PHONE, kind: "phone" },],deploymentRequired: true,});diff --git a/apps/benchmarks/lib/setups/mock-imessage-setup/package.json b/apps/benchmarks/lib/setups/mock-imessage-setup/package.jsonindex 683935e76..48d4da80c 100644--- a/apps/benchmarks/lib/setups/mock-imessage-setup/package.json+++ b/apps/benchmarks/lib/setups/mock-imessage-setup/package.json@@ -1,9 +1,9 @@{- "name": "@photon-ai/setup",+ "name": "@eve-internal/mock-imessage-setup","version": "0.0.0","private": true,"bin": {- "photon-setup": "./cli.mjs"+ "mock-imessage-setup": "./cli.mjs"},"type": "module"}diff --git a/apps/benchmarks/package.json b/apps/benchmarks/package.jsonindex 6d9f8257e..f02db7cd1 100644--- a/apps/benchmarks/package.json+++ b/apps/benchmarks/package.json@@ -5,7 +5,9 @@"type": "module","scripts": {"benchmark": "node run.mjs",- "test": "node --test lib/*.test.mjs",+ "benchmark:publish": "node publish.mjs",+ "benchmark:export": "node scripts/export-results.mjs",+ "test": "node --test lib/*.test.mjs scripts/*.test.mjs","typecheck": "tsc --noEmit"},"dependencies": {@@ -13,7 +15,8 @@"@ai-sdk/harness-codex": "1.0.69","@ai-sdk/sandbox-vercel": "1.0.67","@vercel/agent-eval": "1.5.0",- "@vercel/sandbox": "catalog:"+ "@vercel/sandbox": "catalog:",+ "jiti": "2.7.0"},"devDependencies": {"@types/node": "catalog:",diff --git a/apps/benchmarks/publish.mjs b/apps/benchmarks/publish.mjsnew file mode 100644index 000000000..c3aa817fc--- /dev/null+++ b/apps/benchmarks/publish.mjs@@ -0,0 +1,200 @@+#!/usr/bin/env node++import { execFileSync, spawnSync } from "node:child_process";+import { existsSync, mkdirSync, readdirSync, rmSync, writeFileSync } from "node:fs";+import { dirname, join, resolve } from "node:path";+import { fileURLToPath } from "node:url";+import { parseArgs } from "node:util";++import {+ authoringTreatments,+ publishedBenchmark,+ publishedExperimentId,+} from "./lib/benchmark-config.ts";+import { revisionSubject } from "./lib/source.mjs";++const appRoot = dirname(fileURLToPath(import.meta.url));+const repositoryRoot = resolve(appRoot, "../..");+const evalsRoot = join(appRoot, "evals");+const experimentsRoot = join(appRoot, "experiments");+const { values } = parseArgs({+ args: process.argv.slice(2),+ options: {+ "allow-dirty": { type: "boolean" },+ dry: { type: "boolean" },+ force: { type: "boolean" },+ revision: { type: "string", default: "origin/main" },+ help: { type: "boolean", short: "h" },+ },+ strict: true,+});++if (values.help) {+ console.log(`Usage:+ pnpm benchmark:publish [--revision origin/main] [--dry]+ pnpm benchmark:publish [--revision <revision>] [--force] [--allow-dirty]`);+ process.exit(0);+}+if (values.dry && values.force) throw new Error("--dry and --force cannot be combined.");+if (!values.dry && !values["allow-dirty"] && !checkCleanWorkingTree()) process.exit(1);+if (values["allow-dirty"]) {+ console.warn(+ "Warning: publishing from a dirty working tree is not reproducible from committed source.",+ );+}++const revision = git(["rev-parse", "--verify", `${values.revision}^{commit}`]).trim();+const subject = revisionSubject(repositoryRoot, revision, "published");+const experimentNames = authoringTreatments.map(publishedExperimentId);++prepareFixtures();+writeExperiments(subject, revision);++console.log(`> eve revision: ${revision}`);+console.log(+ `> model: ${publishedBenchmark.modelDisplayName} through ${publishedBenchmark.harness}`,+);+console.log(`> treatments: ${authoringTreatments.join(", ")}`);+console.log(`> runs per cell: ${publishedBenchmark.runs}`);++const executable = join(appRoot, "node_modules/.bin/agent-eval");+if (values.dry) {+ runAgentEval(["status", ...experimentNames]);+ process.exit(0);+}++const cells = experimentNames.flatMap((experiment) =>+ fixtureNames().map((caseId) => ({ experiment, caseId })),+);+for (const [index, cell] of cells.entries()) {+ console.log(`\n> cell ${index + 1}/${cells.length}: ${cell.experiment} / ${cell.caseId}`);+ runAgentEval(+ ["run", cell.experiment, ...(values.force ? ["--force"] : [])],+ { EVE_BENCHMARK_EVAL: cell.caseId },+ false,+ );+}++const pending = benchmarkStatus();+if (pending.totalRun > 0) {+ console.error(+ `\nCannot publish: ${pending.totalRun} benchmark cell(s) still need a valid result.`,+ );+ console.error("Fix any infrastructure failures, then run pnpm benchmark:publish again.");+ process.exit(1);+}++const exported = spawnSync(+ process.execPath,+ [join(appRoot, "scripts/export-results.mjs"), "--revision", revision],+ { cwd: appRoot, stdio: "inherit", env: process.env },+);+if (exported.error) throw exported.error;+process.exit(exported.status ?? 1);++function runAgentEval(args, extraEnv = {}, exitOnFailure = true) {+ const result = spawnSync(executable, args, {+ cwd: appRoot,+ stdio: "inherit",+ env: { ...process.env, ...extraEnv },+ });+ if (result.error) throw result.error;+ if (exitOnFailure && result.status !== 0) process.exit(result.status ?? 1);+ return result.status ?? 1;+}++function benchmarkStatus() {+ const result = spawnSync(executable, ["status", ...experimentNames, "--json"], {+ cwd: appRoot,+ encoding: "utf8",+ env: process.env,+ });+ if (result.error) throw result.error;+ if (result.status !== 0) {+ process.stderr.write(result.stderr);+ process.exit(result.status ?? 1);+ }+ return JSON.parse(result.stdout);+}++function checkCleanWorkingTree() {+ const entries = git([+ "status",+ "--porcelain",+ "--untracked-files=all",+ "--",+ ".",+ ":!pnpm-lock.yaml",+ ])+ .trimEnd()+ .split("\n")+ .filter(Boolean);+ if (entries.length === 0) return true;++ const shown = entries.slice(0, 12);+ console.error("Cannot publish canonical benchmark results from a dirty working tree.\n");+ console.error("Blocking changes:");+ for (const entry of shown) console.error(` ${entry}`);+ if (entries.length > shown.length) {+ console.error(` … and ${entries.length - shown.length} more`);+ }+ console.error(`+Publishing records results against committed source so another run can reproduce them.+Commit, stash, or remove these changes, then run:++ pnpm benchmark:publish++To bypass this check for a local run:++ pnpm benchmark:publish --allow-dirty++To inspect pending benchmark cells without publishing:++ pnpm benchmark:publish --dry`);+ return false;+}++function prepareFixtures() {+ for (const name of fixtureNames()) {+ const fixtureRoot = join(evalsRoot, name);+ writeFileSync(join(fixtureRoot, "PROMPT.md"), "");+ writeFileSync(+ join(fixtureRoot, "package.json"),+ `${JSON.stringify({ name: `eve-authoring-${name}`, private: true, type: "module" }, null, 2)}\n`,+ );+ }+}++function fixtureNames() {+ return readdirSync(evalsRoot, { withFileTypes: true })+ .filter((entry) => entry.isDirectory() && existsSync(join(evalsRoot, entry.name, "CASE.ts")))+ .map((entry) => entry.name)+ .sort();+}++function writeExperiments(subject, revision) {+ rmSync(experimentsRoot, { recursive: true, force: true });+ mkdirSync(experimentsRoot, { recursive: true });+ const archiveName = `published-${revision.slice(0, 12)}.source.tar.gz`;+ writeFileSync(join(experimentsRoot, archiveName), subject.archive);++ for (const treatment of authoringTreatments) {+ const experimentName = publishedExperimentId(treatment);+ writeFileSync(+ join(experimentsRoot, `${experimentName}.ts`),+ `import { readFileSync } from "node:fs";\n` ++ `import { authoringExperiment } from "../lib/experiment.js";\n\n` ++ `export default authoringExperiment({\n` ++ ` archive: readFileSync(new URL(${JSON.stringify(`./${archiveName}`)}, import.meta.url)),\n` ++ ` digest: ${JSON.stringify(subject.digest)},\n` ++ ` dependencyDigest: ${JSON.stringify(subject.dependencyDigest)},\n` ++ ` runs: ${publishedBenchmark.runs},\n` ++ ` treatment: ${JSON.stringify(treatment)},\n` ++ `});\n`,+ );+ }+}++function git(args) {+ return execFileSync("git", args, { cwd: repositoryRoot, encoding: "utf8" });+}diff --git a/apps/benchmarks/run.mjs b/apps/benchmarks/run.mjsindex 6860024b5..36ff3071a 100644--- a/apps/benchmarks/run.mjs+++ b/apps/benchmarks/run.mjs@@ -6,6 +6,7 @@ import { dirname, join, resolve } from "node:path";import { fileURLToPath } from "node:url";import { parseArgs } from "node:util";+import { parseAuthoringTreatment } from "./lib/benchmark-config.ts";import { revisionSubject, workingTreeSubject } from "./lib/source.mjs";const appRoot = dirname(fileURLToPath(import.meta.url));@@ -20,6 +21,7 @@ const { values, positionals } = parseArgs({head: { type: "string" },dry: { type: "boolean" },runs: { type: "string" },+ treatment: { type: "string", default: "guided" },verbose: { type: "boolean" },help: { type: "boolean", short: "h" },},@@ -36,6 +38,7 @@ if (values.head !== undefined && values.base === undefined) {}const runs = parseRuns(values.runs);const selectedEval = positionals[0];+const treatment = parseAuthoringTreatment(values.treatment);if (values.verbose && (selectedEval === undefined || runs !== 1 || values.base !== undefined)) {throw new Error("--verbose requires one eval, one run, and no revision comparison.");}@@ -56,7 +59,7 @@ const subjects =prepareFixtures();mkdirSync(join(appRoot, "results"), { recursive: true });-writeExperiments(subjects, runs, values.verbose ?? false);+writeExperiments(subjects, runs, treatment, values.verbose ?? false);const executable = join(appRoot, "node_modules/.bin/agent-eval");const experimentNames = subjects.map((subject) => subject.label);const args = values.dry ? ["status", ...experimentNames] : ["run", ...experimentNames, "--force"];@@ -88,7 +91,7 @@ function fixtureNames() {.map((entry) => entry.name);}-function writeExperiments(subjects, runs, verbose) {+function writeExperiments(subjects, runs, treatment, verbose) {rmSync(experimentsRoot, { recursive: true, force: true });mkdirSync(experimentsRoot, { recursive: true });for (const subject of subjects) {@@ -103,6 +106,7 @@ function writeExperiments(subjects, runs, verbose) {` digest: ${JSON.stringify(subject.digest)},\n` +` dependencyDigest: ${JSON.stringify(subject.dependencyDigest)},\n` +` runs: ${runs},\n` ++ ` treatment: ${JSON.stringify(treatment)},\n` +` verbose: ${verbose},\n` +`});\n`,);@@ -119,6 +123,6 @@ function parseRuns(value) {function usage() {console.log(`Usage:- pnpm benchmark [eval-name] [--runs N] [--dry] [--verbose]- pnpm benchmark [eval-name] --base <revision> [--head <revision>] [--runs N] [--dry]`);+ pnpm benchmark [eval-name] [--runs N] [--treatment baseline|guided] [--dry] [--verbose]+ pnpm benchmark [eval-name] --base <revision> [--head <revision>] [--runs N] [--treatment baseline|guided] [--dry]`);}diff --git a/apps/benchmarks/scripts/export-results.mjs b/apps/benchmarks/scripts/export-results.mjsnew file mode 100644index 000000000..38de77550--- /dev/null+++ b/apps/benchmarks/scripts/export-results.mjs@@ -0,0 +1,159 @@+#!/usr/bin/env node++import { execFileSync } from "node:child_process";+import { createHash } from "node:crypto";+import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";+import { dirname, join, resolve } from "node:path";+import { fileURLToPath } from "node:url";+import { parseArgs } from "node:util";++import {+ authoringTreatments,+ publishedBenchmark,+ publishedExperimentId,+} from "../lib/benchmark-config.ts";++const appRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");+const repositoryRoot = resolve(appRoot, "../..");+const evalsRoot = join(appRoot, "evals");+const resultsRoot = join(appRoot, "results");+const outputPath = join(repositoryRoot, "apps/docs/lib/evals/benchmark-results.json");+const { values } = parseArgs({+ args: process.argv.slice(2),+ options: {+ revision: { type: "string" },+ output: { type: "string" },+ help: { type: "boolean", short: "h" },+ },+ strict: true,+});++if (values.help) {+ console.log("Usage: node scripts/export-results.mjs --revision <full-sha> [--output <path>]");+ process.exit(0);+}+if (values.revision === undefined || !/^[0-9a-f]{40}$/u.test(values.revision)) {+ throw new Error("--revision must be a full 40-character Git commit SHA.");+}++const caseIds = canonicalCaseIds();+const experimentIds = authoringTreatments.map(publishedExperimentId);+const stale = staleCells(experimentIds);+const results = [];++for (const treatment of authoringTreatments) {+ const experimentId = publishedExperimentId(treatment);+ for (const caseId of caseIds) {+ const staleStatus = stale.get(experimentId);+ const status = staleStatus?.changed.has(caseId)+ ? "stale"+ : staleStatus?.new.has(caseId)+ ? "missing"+ : undefined;+ const measured = latestValidResult(experimentId, caseId);+ if (status !== undefined || measured === undefined) {+ results.push({ experimentId, caseId, status: status ?? "missing" });+ continue;+ }+ results.push({ experimentId, caseId, status: "current", ...measured });+ }+}++const output = {+ schemaVersion: 1,+ generatedAt: new Date().toISOString(),+ suite: {+ eveRevision: values.revision,+ caseFingerprint: caseFingerprint(caseIds),+ caseCount: caseIds.length,+ runsPerCell: publishedBenchmark.runs,+ },+ experiments: authoringTreatments.map((treatment) => ({+ id: publishedExperimentId(treatment),+ groupId: publishedBenchmark.groupId,+ model: publishedBenchmark.model,+ modelDisplayName: publishedBenchmark.modelDisplayName,+ harness: publishedBenchmark.harness,+ treatment,+ })),+ results,+};++const destination =+ values.output === undefined ? outputPath : resolve(process.cwd(), values.output);+mkdirSync(dirname(destination), { recursive: true });+writeFileSync(destination, `${JSON.stringify(output, null, 2)}\n`);+console.log(`Exported ${results.length} benchmark cells to ${destination}`);++function caseFingerprint(caseIds) {+ const hash = createHash("sha256");+ for (const caseId of caseIds) {+ const caseRoot = join(evalsRoot, caseId);+ for (const path of findFiles(caseRoot).sort()) {+ const relativePath = path.slice(caseRoot.length + 1);+ if (relativePath === "PROMPT.md" || relativePath === "package.json") continue;+ hash.update(caseId).update("\0").update(relativePath).update("\0");+ hash.update(readFileSync(path)).update("\0");+ }+ }+ return hash.digest("hex");+}++function canonicalCaseIds() {+ return readdirSync(evalsRoot, { withFileTypes: true })+ .filter((entry) => entry.isDirectory() && existsSync(join(evalsRoot, entry.name, "CASE.ts")))+ .map((entry) => entry.name)+ .sort();+}++function staleCells(experimentIds) {+ const executable = join(appRoot, "node_modules/.bin/agent-eval");+ const raw = execFileSync(executable, ["status", ...experimentIds, "--json"], {+ cwd: appRoot,+ encoding: "utf8",+ maxBuffer: 64 * 1024 * 1024,+ });+ const parsed = JSON.parse(raw);+ const work = Array.isArray(parsed.work) ? parsed.work : [];+ return new Map(+ work.map((entry) => [+ entry.experiment,+ {+ new: new Set(Array.isArray(entry.new) ? entry.new : []),+ changed: new Set(Array.isArray(entry.changed) ? entry.changed : []),+ },+ ]),+ );+}++function latestValidResult(experimentId, caseId) {+ const experimentRoot = join(resultsRoot, experimentId);+ if (!existsSync(experimentRoot)) return undefined;+ const summaries = findFiles(experimentRoot, "summary.json")+ .filter((path) => path.split("/").includes(caseId))+ .sort((left, right) => statSync(right).mtimeMs - statSync(left).mtimeMs);++ for (const summaryPath of summaries) {+ const summary = JSON.parse(readFileSync(summaryPath, "utf8"));+ if (summary.valid === false) continue;+ if (!Number.isFinite(summary.totalRuns) || summary.totalRuns < 1) continue;+ if (!Number.isFinite(summary.passedRuns) || !Number.isFinite(summary.meanDuration)) continue;+ return {+ passedRuns: summary.passedRuns,+ validRuns: summary.totalRuns,+ meanDurationMs: Math.round(summary.meanDuration * 1000),+ measuredAt: statSync(summaryPath).mtime.toISOString(),+ };+ }+ return undefined;+}++function findFiles(root, fileName) {+ const matches = [];+ for (const entry of readdirSync(root, { withFileTypes: true })) {+ const path = join(root, entry.name);+ if (entry.isDirectory()) matches.push(...findFiles(path, fileName));+ else if (fileName === undefined || entry.name === fileName) matches.push(path);+ }+ return matches;+}diff --git a/apps/benchmarks/scripts/export-results.test.mjs b/apps/benchmarks/scripts/export-results.test.mjsnew file mode 100644index 000000000..76e263a25--- /dev/null+++ b/apps/benchmarks/scripts/export-results.test.mjs@@ -0,0 +1,45 @@+import assert from "node:assert/strict";+import { existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { test } from "node:test";+import { execFileSync } from "node:child_process";++const appRoot = new URL("..", import.meta.url);++test("exports missing cells without publishing private artifacts", () => {+ const directory = mkdtempSync(join(tmpdir(), "eve-benchmark-export-"));+ const outputPath = join(directory, "results.json");+ const resultsPath = new URL("../results", import.meta.url).pathname;+ const savedResultsPath = join(directory, "saved-results");+ try {+ if (existsSync(resultsPath)) renameSync(resultsPath, savedResultsPath);+ mkdirSync(resultsPath, { recursive: true });+ execFileSync(+ process.execPath,+ [+ new URL("./export-results.mjs", import.meta.url).pathname,+ "--revision",+ "a".repeat(40),+ "--output",+ outputPath,+ ],+ { cwd: appRoot, stdio: "pipe" },+ );+ const raw = readFileSync(outputPath, "utf8");+ const output = JSON.parse(raw);++ assert.equal(output.schemaVersion, 1);+ assert.equal(output.suite.caseCount, 3);+ assert.match(output.suite.caseFingerprint, /^[0-9a-f]{64}$/u);+ assert.equal(output.results.length, 6);+ assert.ok(output.results.every((result) => result.status === "missing"));+ for (const privateField of ["transcript", "commands", "worldEvents", "files"]) {+ assert.equal(raw.includes(`\"${privateField}\"`), false);+ }+ } finally {+ rmSync(resultsPath, { recursive: true, force: true });+ if (existsSync(savedResultsPath)) renameSync(savedResultsPath, resultsPath);+ rmSync(directory, { recursive: true, force: true });+ }+});diff --git a/package.json b/package.jsonindex d76d18c49..c7ec46db7 100644--- a/package.json+++ b/package.json@@ -13,6 +13,7 @@"type": "module","scripts": {"benchmark": "pnpm --filter @eve-internal/benchmarks benchmark",+ "benchmark:publish": "node apps/benchmarks/publish.mjs","build": "turbo run build --filter='./packages/*'","changeset": "changeset","check:deps": "syncpack lint",diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex a3fde8db2..922745dff 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -146,6 +146,9 @@ importers:'@vercel/sandbox':specifier: 'catalog:'version: 2.8.0+ jiti:+ specifier: 2.7.0+ version: 2.7.0devDependencies:'@types/node':specifier: 'catalog:'