The Open Framework for Building Agents
perf: profile authoring benchmark setup (#2219)
OwenKephart085aaee99913
diff --git a/apps/benchmarks/README.md b/apps/benchmarks/README.mdindex 2910939e2..0f4ee6bd9 100644--- a/apps/benchmarks/README.md+++ b/apps/benchmarks/README.md@@ -35,15 +35,26 @@ pnpm benchmark author-000-imessage \```The runner archives each subject locally and uploads it to the sandbox. Revisions and local-only-commits do not need to be pushed. Dependency downloads are cached by lockfile, so source-only-changes reuse the prepared pnpm store. For one eval and one run, `--verbose` streams setup-phases, assistant text, tool calls, grading, and build progress.+commits do not need to be pushed. It maintains two persistent snapshot layers: a dependency+snapshot keyed by package-manager inputs, and a subject snapshot keyed by the source tree,+starting point, setup IDs, and bootstrap version. Source-only changes reuse the dependency snapshot+but create a new subject snapshot. For one eval and one run, `--verbose` streams setup phases,+assistant text, tool calls, grading, and build progress.Local runs use the `guided` treatment by default, which keeps the `AGENTS.md` and aliases generatedby `eve init`. Pass `--treatment baseline` to remove those files before the coding agent starts.Results are written under `apps/benchmarks/results/`. Each run includes the transcript,-grader output, summary, and copied project files. Vercel Sandbox and AI Gateway credentials are+grader output, summary, copied project files, and `project/benchmark/timings.json`. The timing+artifact records the snapshot-cache outcome, source installation and build phases, workspace setup,+each user turn with token and tool-call counts, and grading and validation durations. Use it to+separate sandbox setup time from agent time when comparing runs. Print a compact local report with:++```sh+pnpm benchmark:timings results/current/<timestamp>/<case>/run-1+```++Pass `--json` to print the original timing artifact. Vercel Sandbox and AI Gateway credentials arerequired.## Publish canonical resultsdiff --git a/apps/benchmarks/lib/dependency-sandbox.test.mjs b/apps/benchmarks/lib/dependency-sandbox.test.mjsdeleted file mode 100644index 89df45999..000000000--- a/apps/benchmarks/lib/dependency-sandbox.test.mjs+++ /dev/null@@ -1,29 +0,0 @@-import assert from "node:assert/strict";-import { test } from "node:test";--import { dependencySnapshotId } from "./dependency-snapshot.ts";--test("reuses a dependency sandbox's current snapshot", async () => {- const snapshotId = await dependencySnapshotId({- currentSnapshotId: "snapshot-current",- snapshot: async () => {- throw new Error("snapshot should not be called");- },- });-- assert.equal(snapshotId, "snapshot-current");-});--test("explicitly snapshots a dependency sandbox without a current snapshot", async () => {- let expiration;- const snapshotId = await dependencySnapshotId({- currentSnapshotId: undefined,- snapshot: async (options) => {- expiration = options?.expiration;- return { snapshotId: "snapshot-created" };- },- });-- assert.equal(snapshotId, "snapshot-created");- assert.equal(expiration, 0);-});diff --git a/apps/benchmarks/lib/dependency-sandbox.ts b/apps/benchmarks/lib/dependency-sandbox.tsindex 88d89ad2d..cb16cf905 100644--- a/apps/benchmarks/lib/dependency-sandbox.ts+++ b/apps/benchmarks/lib/dependency-sandbox.ts@@ -3,16 +3,19 @@ import { createVercelSandbox } from "@ai-sdk/sandbox-vercel";import { Sandbox } from "@vercel/sandbox";import { SOURCE_ARCHIVE_PATH, SOURCE_ROOT } from "./paths.js";+import type { BenchmarkTimings } from "./timing.js";const dependencySnapshots = new Map<string, Promise<string>>();const subjectSnapshots = new Map<string, Promise<string>>();export function createDependencyCachedSandbox(options: {readonly archive: Uint8Array;+ readonly dependencyArchive: Uint8Array;readonly dependencyDigest: string;readonly ports: ReadonlyArray<number>;readonly env: Readonly<Record<string, string>>;readonly log: (message: string) => void;+ readonly timings: BenchmarkTimings;}): HarnessV1SandboxProvider {const sessionProvider = (snapshotId: string) =>createVercelSandbox({@@ -29,19 +32,22 @@ export function createDependencyCachedSandbox(options: {async createSession(request = {}) {options.log("[setup] preparing dependency cache");const dependencies = await dependencySnapshot(- options.archive,+ options.dependencyArchive,options.dependencyDigest,options.log,+ options.timings,);if (request.identity === undefined || request.onFirstCreate === undefined) {return sessionProvider(dependencies).createSession(request);}const subject = await subjectSnapshot(dependencies,+ options.archive,request.identity,options.env,request.onFirstCreate,request.abortSignal,+ options.timings,);return sessionProvider(subject).createSession({sessionId: request.sessionId,@@ -50,9 +56,10 @@ export function createDependencyCachedSandbox(options: {},async resumeSession(request) {const dependencies = await dependencySnapshot(- options.archive,+ options.dependencyArchive,options.dependencyDigest,options.log,+ options.timings,);const provider = sessionProvider(dependencies);if (provider.resumeSession === undefined) {@@ -67,11 +74,15 @@ function dependencySnapshot(archive: Uint8Array,digest: string,log: (message: string) => void,+ timings: BenchmarkTimings,): Promise<string> {- const name = `eve-benchmark-dependencies-v4-${digest.slice(0, 24)}`;+ const name = `eve-benchmark-dependencies-v5-${digest.slice(0, 24)}`;let snapshot = dependencySnapshots.get(name);- if (snapshot !== undefined) return snapshot;- snapshot = createDependencySnapshot(name, archive, log);+ if (snapshot !== undefined) {+ timings.record("dependency-snapshot.memory-cache", 0, "success", { name });+ return snapshot;+ }+ snapshot = createDependencySnapshot(name, archive, log, timings);dependencySnapshots.set(name, snapshot);return snapshot;}@@ -80,49 +91,70 @@ async function createDependencySnapshot(name: string,archive: Uint8Array,log: (message: string) => void,+ timings: BenchmarkTimings,): Promise<string> {let created = false;- const sandbox = await Sandbox.getOrCreate({- name,- runtime: "node24",- timeout: 15 * 60_000,- persistent: true,- snapshotExpiration: 0,- networkPolicy: "allow-all",- async onCreate(current) {- created = true;- log("[setup] fetching workspace dependencies");- await current.writeFiles([{ path: SOURCE_ARCHIVE_PATH, content: archive }]);- const command = await current.runCommand("bash", [- "-lc",- `mkdir -p ${SOURCE_ROOT} && tar -xzf ${SOURCE_ARCHIVE_PATH} -C ${SOURCE_ROOT} && npm install --global [email protected] [email protected] && cd ${SOURCE_ROOT} && pnpm fetch --frozen-lockfile`,- ]);- if (command.exitCode !== 0) {- throw new Error(- `Dependency setup failed (${command.exitCode}):\n${await command.stdout()}\n${await command.stderr()}`,- );- }- },- });- if (!created && sandbox.currentSnapshotId !== undefined) return sandbox.currentSnapshotId;- return stopWithSnapshot(sandbox, "Dependency");+ const sandbox = await timings.measure("dependency-snapshot.get-or-create", () =>+ Sandbox.getOrCreate({+ name,+ runtime: "node24",+ timeout: 15 * 60_000,+ persistent: true,+ snapshotExpiration: 0,+ networkPolicy: "allow-all",+ async onCreate(current) {+ created = true;+ log("[setup] fetching workspace dependencies");+ await current.writeFiles([{ path: SOURCE_ARCHIVE_PATH, content: archive }]);+ const command = await current.runCommand("bash", [+ "-lc",+ `mkdir -p ${SOURCE_ROOT} && tar -xzf ${SOURCE_ARCHIVE_PATH} -C ${SOURCE_ROOT} && npm install --global [email protected] [email protected] && cd ${SOURCE_ROOT} && pnpm fetch --frozen-lockfile`,+ ]);+ if (command.exitCode !== 0) {+ throw new Error(+ `Dependency setup failed (${command.exitCode}):\n${await command.stdout()}\n${await command.stderr()}`,+ );+ }+ },+ }),+ );+ if (!created && sandbox.currentSnapshotId !== undefined) {+ timings.record("dependency-snapshot.reused", 0, "success", { name });+ return sandbox.currentSnapshotId;+ }+ return timings.measure("dependency-snapshot.publish", () =>+ stopWithSnapshot(sandbox, "Dependency"),+ );}// Publish bootstrap mutations explicitly; a layered Vercel template can expose// its inherited snapshot ID before those mutations receive a new snapshot.function subjectSnapshot(dependencySnapshotId: string,+ archive: Uint8Array,identity: string,env: Readonly<Record<string, string>>,bootstrap: NonNullable<NonNullable<Parameters<HarnessV1SandboxProvider["createSession"]>[0]>["onFirstCreate"]>,- abortSignal?: AbortSignal,+ abortSignal: AbortSignal | undefined,+ timings: BenchmarkTimings,): Promise<string> {const name = `eve-benchmark-subject-${identity}`;let snapshot = subjectSnapshots.get(name);- if (snapshot !== undefined) return snapshot;- snapshot = createSubjectSnapshot(name, dependencySnapshotId, env, bootstrap, abortSignal);+ if (snapshot !== undefined) {+ timings.record("subject-snapshot.memory-cache", 0, "success", { name });+ return snapshot;+ }+ snapshot = createSubjectSnapshot(+ name,+ dependencySnapshotId,+ archive,+ env,+ bootstrap,+ abortSignal,+ timings,+ );subjectSnapshots.set(name, snapshot);return snapshot;}@@ -130,31 +162,41 @@ function subjectSnapshot(async function createSubjectSnapshot(name: string,dependencySnapshotId: string,+ archive: Uint8Array,env: Readonly<Record<string, string>>,bootstrap: NonNullable<NonNullable<Parameters<HarnessV1SandboxProvider["createSession"]>[0]>["onFirstCreate"]>,- abortSignal?: AbortSignal,+ abortSignal: AbortSignal | undefined,+ timings: BenchmarkTimings,): Promise<string> {let created = false;- const sandbox = await Sandbox.getOrCreate({- name,- source: { type: "snapshot", snapshotId: dependencySnapshotId },- timeout: 15 * 60_000,- env: { ...env },- persistent: true,- snapshotExpiration: 0,- networkPolicy: "allow-all",- signal: abortSignal,- async onCreate(current) {- created = true;- const provider = createVercelSandbox({ sandbox: current });- const session = await provider.createSession({ abortSignal });- await bootstrap(session.restricted(), { abortSignal });- },- });- if (!created && sandbox.currentSnapshotId !== undefined) return sandbox.currentSnapshotId;- return stopWithSnapshot(sandbox, "Subject", abortSignal);+ const sandbox = await timings.measure("subject-snapshot.get-or-create", () =>+ Sandbox.getOrCreate({+ name,+ source: { type: "snapshot", snapshotId: dependencySnapshotId },+ timeout: 15 * 60_000,+ env: { ...env },+ persistent: true,+ snapshotExpiration: 0,+ networkPolicy: "allow-all",+ signal: abortSignal,+ async onCreate(current) {+ created = true;+ await current.writeFiles([{ path: SOURCE_ARCHIVE_PATH, content: archive }]);+ const provider = createVercelSandbox({ sandbox: current });+ const session = await provider.createSession({ abortSignal });+ await bootstrap(session.restricted(), { abortSignal });+ },+ }),+ );+ if (!created && sandbox.currentSnapshotId !== undefined) {+ timings.record("subject-snapshot.reused", 0, "success", { name });+ return sandbox.currentSnapshotId;+ }+ return timings.measure("subject-snapshot.publish", () =>+ stopWithSnapshot(sandbox, "Subject", abortSignal),+ );}async function stopWithSnapshot(diff --git a/apps/benchmarks/lib/dependency-snapshot.ts b/apps/benchmarks/lib/dependency-snapshot.tsdeleted file mode 100644index 8ae18e473..000000000--- a/apps/benchmarks/lib/dependency-snapshot.ts+++ /dev/null@@ -1,8 +0,0 @@-import type { Sandbox } from "@vercel/sandbox";--export async function dependencySnapshotId(- sandbox: Pick<Sandbox, "currentSnapshotId" | "snapshot">,-): Promise<string> {- if (sandbox.currentSnapshotId !== undefined) return sandbox.currentSnapshotId;- return (await sandbox.snapshot({ expiration: 0 })).snapshotId;-}diff --git a/apps/benchmarks/lib/experiment-files.mjs b/apps/benchmarks/lib/experiment-files.mjsnew file mode 100644index 000000000..27c9d9a65--- /dev/null+++ b/apps/benchmarks/lib/experiment-files.mjs@@ -0,0 +1,52 @@+import { existsSync, mkdirSync, readdirSync, rmSync, writeFileSync } from "node:fs";+import { join } from "node:path";++export function fixtureNames(evalsRoot) {+ return readdirSync(evalsRoot, { withFileTypes: true })+ .filter((entry) => entry.isDirectory() && existsSync(join(evalsRoot, entry.name, "CASE.ts")))+ .map((entry) => entry.name)+ .sort();+}++export function prepareFixtures(evalsRoot, names = fixtureNames(evalsRoot)) {+ for (const name of names) {+ const fixtureRoot = join(evalsRoot, name);+ writeFileSync(join(fixtureRoot, "PROMPT.md"), "");+ writeFileSync(+ join(fixtureRoot, "package.json"),+ `${JSON.stringify({ name: `eve-authoring-${name}`, private: true, type: "module" }, null, 2)}\n`,+ );+ }+}++export function resetExperiments(experimentsRoot) {+ rmSync(experimentsRoot, { recursive: true, force: true });+ mkdirSync(experimentsRoot, { recursive: true });+}++export function writeSubjectArchives(experimentsRoot, subject, name) {+ const archiveName = `${name}.source.tar.gz`;+ const dependencyArchiveName = `${name}.dependencies.tar.gz`;+ writeFileSync(join(experimentsRoot, archiveName), subject.archive);+ writeFileSync(join(experimentsRoot, dependencyArchiveName), subject.dependencyArchive);+ return { archiveName, dependencyArchiveName };+}++export function writeExperiment(experimentsRoot, name, options) {+ writeFileSync(+ join(experimentsRoot, `${name}.ts`),+ `import { readFileSync } from "node:fs";\n` ++ `import { authoringExperiment } from "../lib/experiment.js";\n\n` ++ `export default authoringExperiment({\n` ++ ` archive: readFileSync(new URL(${JSON.stringify(`./${options.archiveName}`)}, import.meta.url)),\n` ++ ` dependencyArchive: readFileSync(new URL(${JSON.stringify(`./${options.dependencyArchiveName}`)}, import.meta.url)),\n` ++ ` digest: ${JSON.stringify(options.digest)},\n` ++ ` dependencyDigest: ${JSON.stringify(options.dependencyDigest)},\n` ++ ` runs: ${options.runs},\n` ++ (options.evals === undefined ? "" : ` evals: ${JSON.stringify(options.evals)},\n`) ++ ` benchmark: ${JSON.stringify(options.benchmark)},\n` ++ ` treatment: ${JSON.stringify(options.treatment)},\n` ++ (options.verbose === undefined ? "" : ` verbose: ${options.verbose},\n`) ++ `});\n`,+ );+}diff --git a/apps/benchmarks/lib/experiment-files.test.mjs b/apps/benchmarks/lib/experiment-files.test.mjsnew file mode 100644index 000000000..b122d3733--- /dev/null+++ b/apps/benchmarks/lib/experiment-files.test.mjs@@ -0,0 +1,77 @@+import assert from "node:assert/strict";+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { test } from "node:test";++import {+ fixtureNames,+ prepareFixtures,+ resetExperiments,+ writeExperiment,+ writeSubjectArchives,+} from "./experiment-files.mjs";++const subject = {+ archive: Buffer.from("source"),+ dependencyArchive: Buffer.from("dependencies"),+ digest: "source-digest",+ dependencyDigest: "dependency-digest",+};+const benchmark = {+ id: "test",+ model: "test/model",+ displayName: "Test",+ harness: "OpenCode",+ support: "supported",+};++test("materializes fixtures and complete experiment inputs", () => {+ const root = mkdtempSync(join(tmpdir(), "eve-benchmark-experiments-"));+ const evals = join(root, "evals");+ const experiments = join(root, "experiments");+ try {+ writeCase(evals, "author-002-second");+ writeCase(evals, "author-001-first");+ writeFileSync(join(evals, "not-a-case"), "ignored");+ assert.deepEqual(fixtureNames(evals), ["author-001-first", "author-002-second"]);++ prepareFixtures(evals);+ assert.equal(readFileSync(join(evals, "author-001-first", "PROMPT.md"), "utf8"), "");+ assert.deepEqual(JSON.parse(readFileSync(join(evals, "author-001-first", "package.json"))), {+ name: "eve-authoring-author-001-first",+ private: true,+ type: "module",+ });++ resetExperiments(experiments);+ const archives = writeSubjectArchives(experiments, subject, "published-deadbeef");+ writeExperiment(experiments, "test-opencode--guided", {+ ...archives,+ digest: subject.digest,+ dependencyDigest: subject.dependencyDigest,+ runs: 3,+ evals: ["author-001-first"],+ benchmark,+ treatment: "guided",+ });++ assert.equal(readFileSync(join(experiments, archives.archiveName), "utf8"), "source");+ assert.equal(+ readFileSync(join(experiments, archives.dependencyArchiveName), "utf8"),+ "dependencies",+ );+ const experiment = readFileSync(join(experiments, "test-opencode--guided.ts"), "utf8");+ assert.match(experiment, /dependencyArchive: readFileSync/u);+ assert.match(experiment, /published-deadbeef\.dependencies\.tar\.gz/u);+ } finally {+ rmSync(root, { recursive: true, force: true });+ }+});++function writeCase(evals, name) {+ const root = join(evals, name);+ mkdirSync(root, { recursive: true });+ writeFileSync(join(root, "CASE.ts"), "export default {};\n");+ assert.ok(existsSync(root));+}diff --git a/apps/benchmarks/lib/experiment.ts b/apps/benchmarks/lib/experiment.tsindex 3dc6094cf..f447286ac 100644--- a/apps/benchmarks/lib/experiment.ts+++ b/apps/benchmarks/lib/experiment.ts@@ -6,6 +6,7 @@ import { createAuthoringAgent } from "./harness-agent.js";export function authoringExperiment(options: {readonly archive: Uint8Array;+ readonly dependencyArchive: Uint8Array;readonly digest: string;readonly dependencyDigest: string;readonly runs?: number;@@ -17,6 +18,7 @@ export function authoringExperiment(options: {const agent = createAuthoringAgent({model: options.benchmark.model,archive: options.archive,+ dependencyArchive: options.dependencyArchive,digest: options.digest,dependencyDigest: options.dependencyDigest,});diff --git a/apps/benchmarks/lib/harness-agent.ts b/apps/benchmarks/lib/harness-agent.tsindex 9fed52f71..8ba587c0d 100644--- a/apps/benchmarks/lib/harness-agent.ts+++ b/apps/benchmarks/lib/harness-agent.ts@@ -23,12 +23,14 @@ import {WORKSPACE,} from "./paths.js";import type { AuthoringTokenUsage, AuthoringTranscriptEntry } from "./protocol.js";+import { BenchmarkTimings } from "./timing.js";const HARNESS_BRIDGE_PORT = 4172;-const BOOTSTRAP_VERSION = "v5";+const BOOTSTRAP_VERSION = "v6";export function createAuthoringAgent(subject: {readonly model: string;readonly archive: Uint8Array;+ readonly dependencyArchive: Uint8Array;readonly digest: string;readonly dependencyDigest: string;}): Agent {@@ -57,8 +59,18 @@ export function createAuthoringAgent(subject: {const setups = [authoringCase.startingPoint.setup, authoringCase.setup].filter((setup): setup is AuthoringSetup => setup !== undefined,);+ const timings = new BenchmarkTimings();+ timings.record("run.context", 0, "success", {+ sourceArchiveBytes: subject.archive.length,+ dependencyArchiveBytes: subject.dependencyArchive.length,+ sourceDigest: subject.digest,+ dependencyDigest: subject.dependencyDigest,+ startingPoint: authoringCase.startingPoint.id,+ setupCount: setups.length,+ });const sandbox = createDependencyCachedSandbox({archive: subject.archive,+ dependencyArchive: subject.dependencyArchive,dependencyDigest: subject.dependencyDigest,ports: [HARNESS_BRIDGE_PORT, ...new Set(setups.flatMap((setup) => setup.ports ?? []))],env: {@@ -67,6 +79,7 @@ export function createAuthoringAgent(subject: {...Object.assign({}, ...setups.map((setup) => setup.environment ?? {})),},log,+ timings,});const startedAt = Date.now();const commands: string[] = [];@@ -89,22 +102,27 @@ export function createAuthoringAgent(subject: {onBootstrap: async ({ session: bootstrap, workDir }) => {const bootstrapSandbox = bootstrap as HarnessV1NetworkSandboxSession;const context = setupContext(bootstrapSandbox, workDir);- for (const setup of setups) await setup.onBootstrap?.(context);+ await timings.measure("subject.case-bootstrap", async () => {+ for (const setup of setups) await setup.onBootstrap?.(context);+ });log("[setup] building the selected eve source");await bootstrapSubject(bootstrapSandbox,workDir,authoringCase.startingPoint.workspace,subject.archive,+ timings,);},onSession: async ({ session: current, sessionWorkDir }) => {activeSandbox = current as HarnessV1NetworkSandboxSession;workspace = sessionWorkDir;const context = setupContext(activeSandbox, workspace);- for (const setup of setups) await setup.onSession?.(context);+ await timings.measure("session.setup", async () => {+ for (const setup of setups) await setup.onSession?.(context);+ if (options.agentOptions?.agentsMd !== true) await installBaselineEveWrapper(context);+ });if (options.agentOptions?.agentsMd !== true) {- await installBaselineEveWrapper(context);await context.run("rm -f AGENTS.md CLAUDE.md GEMINI.md");}},@@ -113,7 +131,7 @@ export function createAuthoringAgent(subject: {});try {- session = await withBootstrapInitialization(bootstrapHash(authoringCase, subject), () =>+ session = await timings.measure("session.create", () =>agent.createSession({ abortSignal: options.signal }),);if (activeSandbox === undefined || workspace === undefined) {@@ -126,22 +144,25 @@ export function createAuthoringAgent(subject: {send: async (prompt) => {transcript.push({ role: "user", content: prompt });if (verbose) console.log(`[user] ${prompt}`);- const result = verbose- ? await streamTurn(agent, session!, prompt, options.timeout, options.signal)- : await agent.generate({- session: session!,- prompt,- timeout: options.timeout,- abortSignal: options.signal,- });- transcript.push({- role: "assistant",- content: result.text,- toolCalls: authoringToolCalls(result.toolCalls),- usage: normalizeUsage(result.usage),+ const turn = transcript.filter((entry) => entry.role === "user").length;+ const result = await timings.measure(`agent.turn.${turn}`, () =>+ verbose+ ? streamTurn(agent, session!, prompt, options.timeout, options.signal)+ : generateTurn(agent, session!, prompt, options.timeout, options.signal),+ );+ const toolCalls = result.toolCalls;+ const usage = result.usage;+ transcript.push({ role: "assistant", content: result.text, toolCalls, usage });+ timings.record(`agent.turn.${turn}.summary`, 0, "success", {+ promptCharacters: prompt.length,+ responseCharacters: result.text.length,+ toolCalls: toolCalls.length,+ inputTokens: usage.inputTokens,+ outputTokens: usage.outputTokens,+ reasoningTokens: usage.reasoningTokens,});commands.push(...shellCommands(result.toolCalls));- return { text: result.text, toolCalls: authoringToolCalls(result.toolCalls) };+ return { text: result.text, toolCalls };},});@@ -178,19 +199,19 @@ export function createAuthoringAgent(subject: {]);log("[grade] running deterministic assertions");- const test = await resultOf(- activeSandbox,- `ln -s ${workspace}/${AGENT_EVAL_DIRECTORY} .eve-grader && trap 'rm -f .eve-grader' EXIT && vitest run .eve-grader/EVAL.test.ts`,- projectWorkspace,+ const test = await timings.measure("validation.grader", () =>+ resultOf(+ activeSandbox!,+ `ln -s ${workspace}/${AGENT_EVAL_DIRECTORY} .eve-grader && trap 'rm -f .eve-grader' EXIT && vitest run .eve-grader/EVAL.test.ts`,+ projectWorkspace,+ ),);const scriptsResults = Object.fromEntries(await Promise.all((options.scripts ?? []).map(async (script) => {log(`[${script}] running`);- const result = await resultOf(- activeSandbox!,- `npm run ${shellQuote(script)}`,- projectWorkspace,+ const result = await timings.measure(`validation.${script}`, () =>+ resultOf(activeSandbox!, `npm run ${shellQuote(script)}`, projectWorkspace),);return [script,@@ -200,6 +221,12 @@ export function createAuthoringAgent(subject: {),);const scriptsPassed = Object.values(scriptsResults).every((result) => result.success);+ timings.record("run.total", Date.now() - startedAt);+ await context.write(+ `${AGENT_EVAL_DIRECTORY}/timings.json`,+ JSON.stringify(timings.entries),+ );+ logTimingSummary(log, timings);log(`[result] ${test.exitCode === 0 && scriptsPassed ? "passed" : "failed"}`);return {success: test.exitCode === 0 && scriptsPassed,@@ -211,12 +238,21 @@ export function createAuthoringAgent(subject: {transcript: harnessTranscript(transcript),scriptsResults,sandboxId: activeSandbox.id,+ generatedFiles: timingArtifact(timings),};} catch (error) {+ timings.record("run.total", Date.now() - startedAt, "failure");if (activeSandbox !== undefined && workspace !== undefined) {- await setupContext(activeSandbox, workspace)- .write(`${AGENT_EVAL_DIRECTORY}/harness-transcript.json`, JSON.stringify(transcript))- .catch(() => undefined);+ await Promise.all([+ setupContext(activeSandbox, workspace).write(+ `${AGENT_EVAL_DIRECTORY}/harness-transcript.json`,+ JSON.stringify(transcript),+ ),+ setupContext(activeSandbox, workspace).write(+ `${AGENT_EVAL_DIRECTORY}/timings.json`,+ JSON.stringify(timings.entries),+ ),+ ]).catch(() => undefined);}const result: AgentRunResult = {success: false,@@ -225,6 +261,7 @@ export function createAuthoringAgent(subject: {duration: Date.now() - startedAt,transcript: harnessTranscript(transcript),scriptsResults: {},+ generatedFiles: timingArtifact(timings),};if (activeSandbox !== undefined) result.sandboxId = activeSandbox.id;return result;@@ -235,6 +272,21 @@ export function createAuthoringAgent(subject: {};}+async function generateTurn(+ agent: HarnessAgent,+ session: HarnessAgentSession,+ prompt: string,+ timeout: number,+ abortSignal?: AbortSignal,+): Promise<AuthoringTurn & { usage: AuthoringTokenUsage }> {+ const result = await agent.generate({ session, prompt, timeout, abortSignal });+ return {+ text: result.text,+ toolCalls: authoringToolCalls(result.toolCalls),+ usage: normalizeUsage(result.usage),+ };+}+async function streamTurn(agent: HarnessAgent,session: HarnessAgentSession,@@ -309,22 +361,25 @@ async function bootstrapSubject(workspace: string,workspaceKind: "scaffolded" | "empty",archive: Uint8Array,+ timings: BenchmarkTimings,): Promise<void> {- await sandbox.writeBinaryFile({ path: SOURCE_ARCHIVE_PATH, content: archive });- await run(- sandbox,- [- `rm -rf ${SOURCE_ROOT} && mkdir -p ${SOURCE_ROOT}`,- `tar -xzf ${SOURCE_ARCHIVE_PATH} -C ${SOURCE_ROOT}`,- `pnpm --dir ${SOURCE_ROOT} install --frozen-lockfile --offline`,- `pnpm --dir ${SOURCE_ROOT} --filter eve build`,- "mkdir -p /tmp/eve-package",- `pnpm --dir ${SOURCE_ROOT}/packages/eve pack --pack-destination /tmp/eve-package`,- `mv $(find /tmp/eve-package -name '*.tgz' -print -quit) ${EVE_PACKAGE_PATH}`,- `npm install --global --package-lock=false ${EVE_PACKAGE_PATH}`,- 'ln -sf "$(npm prefix --global)/bin/eve" /usr/local/bin/eve',- "command -v eve",- ].join(" && "),+ await timings.measure("subject.source-upload", async () => {+ await sandbox.writeBinaryFile({ path: SOURCE_ARCHIVE_PATH, content: archive });+ });+ await timings.measure("subject.source-install", () =>+ run(+ sandbox,+ `rm -rf ${SOURCE_ROOT} && mkdir -p ${SOURCE_ROOT} && tar -xzf ${SOURCE_ARCHIVE_PATH} -C ${SOURCE_ROOT} && pnpm --dir ${SOURCE_ROOT} install --frozen-lockfile --offline`,+ ),+ );+ await timings.measure("subject.eve-build", () =>+ run(sandbox, `pnpm --dir ${SOURCE_ROOT} --filter eve build`),+ );+ await timings.measure("subject.eve-pack-and-cli", () =>+ run(+ sandbox,+ `mkdir -p /tmp/eve-package && pnpm --dir ${SOURCE_ROOT}/packages/eve pack --pack-destination /tmp/eve-package && mv $(find /tmp/eve-package -name '*.tgz' -print -quit) ${EVE_PACKAGE_PATH} && ln -sf ${SOURCE_ROOT}/packages/eve/bin/eve.js /usr/local/bin/eve && command -v eve`,+ ),);const artifactsRoot = `${workspace}/${AGENT_EVAL_DIRECTORY}`;const workspaceCommands: string[] = [];@@ -339,7 +394,9 @@ async function bootstrapSubject(if (workspaceKind === "scaffolded") {workspaceCommands.push(`test -f ${workspace}/package.json`);}- await run(sandbox, workspaceCommands.join(" && "));+ await timings.measure("subject.workspace-bootstrap", () =>+ run(sandbox, workspaceCommands.join(" && ")),+ );}function setupContext(@@ -364,7 +421,7 @@ function setupContext(async function installBaselineEveWrapper(context: AuthoringSetupContext): Promise<void> {await context.run(`-cli_path="$(npm prefix --global)/bin/eve"+cli_path="/usr/local/bin/eve"test -x "$cli_path"real_cli="$cli_path.guided"if [ ! -e "$real_cli" ]; then mv "$cli_path" "$real_cli"; fi@@ -379,6 +436,16 @@ chmod +x "$cli_path"`);}+function timingArtifact(timings: BenchmarkTimings): Record<string, string> {+ return { "benchmark/timings.json": `${JSON.stringify(timings.entries, null, 2)}\n` };+}++function logTimingSummary(log: (message: string) => void, timings: BenchmarkTimings): void {+ for (const timing of timings.entries) {+ log(`[timing] ${timing.phase}: ${timing.durationMs}ms (${timing.outcome})`);+ }+}+function harnessTranscript(transcript: ReadonlyArray<AuthoringTranscriptEntry>): string {return transcript.map((entry) => {@@ -422,43 +489,6 @@ function bootstrapHash(authoringCase: AuthoringCase, subject: { readonly digest:return `eve-authoring-${BOOTSTRAP_VERSION}-${subject.digest}-${authoringCase.startingPoint.id}-${setupIds}`;}-const bootstrapCoordination = globalThis as typeof globalThis & {- __eveAuthoringBootstrapLocks?: Map<string, Promise<void>>;- __eveAuthoringBootstrapsReady?: Set<string>;-};--async function withBootstrapInitialization<T>(- key: string,- operation: () => Promise<T>,-): Promise<T> {- const ready = (bootstrapCoordination.__eveAuthoringBootstrapsReady ??= new Set());- if (ready.has(key)) return operation();-- const locks = (bootstrapCoordination.__eveAuthoringBootstrapLocks ??= new Map());- const previous = locks.get(key) ?? Promise.resolve();- let release = () => {};- const current = new Promise<void>((resolve) => {- release = resolve;- });- const tail = previous.then(() => current);- locks.set(key, tail);-- await previous;- if (ready.has(key)) {- release();- if (locks.get(key) === tail) locks.delete(key);- return operation();- }- try {- const result = await operation();- ready.add(key);- return result;- } finally {- release();- if (locks.get(key) === tail) locks.delete(key);- }-}-async function resultOf(sandbox: HarnessV1NetworkSandboxSession,command: string,diff --git a/apps/benchmarks/lib/load-authoring-case.test.mjs b/apps/benchmarks/lib/load-authoring-case.test.mjsindex 4d9659e13..467e2b18a 100644--- a/apps/benchmarks/lib/load-authoring-case.test.mjs+++ b/apps/benchmarks/lib/load-authoring-case.test.mjs@@ -1,5 +1,5 @@import assert from "node:assert/strict";-import { readdir } from "node:fs/promises";+import { readdir, stat } from "node:fs/promises";import { dirname, resolve } from "node:path";import test from "node:test";import { fileURLToPath } from "node:url";@@ -12,7 +12,7 @@ const jiti = createJiti(import.meta.url, { interopDefault: true, moduleCache: faconst { loadAuthoringCase } = await jiti.import(resolve(libRoot, "load-authoring-case.ts"));for (const entry of await readdir(evalsRoot, { withFileTypes: true })) {- if (!entry.isDirectory()) continue;+ if (!entry.isDirectory() || !(await hasCase(entry.name))) continue;test(`loads ${entry.name}/CASE.ts`, async () => {const authoringCase = await loadAuthoringCase(resolve(evalsRoot, entry.name));@@ -21,6 +21,14 @@ for (const entry of await readdir(evalsRoot, { withFileTypes: true })) {});}+async function hasCase(name) {+ try {+ return (await stat(resolve(evalsRoot, name, "CASE.ts"))).isFile();+ } catch {+ return false;+ }+}+test("loads the named project output directory", async () => {const authoringCase = await loadAuthoringCase(resolve(evalsRoot, "author-002-new-project"));assert.equal(authoringCase.projectDirectory, "wayfinder");diff --git a/apps/benchmarks/lib/paths.ts b/apps/benchmarks/lib/paths.tsindex d1455fea8..f14c9ff80 100644--- a/apps/benchmarks/lib/paths.ts+++ b/apps/benchmarks/lib/paths.ts@@ -1,10 +1,8 @@export const WORKSPACE = "workspace";-export const AGENT_EVAL_DIRECTORY = "/tmp/eve-grader";-export const WORKSPACE_ENV = "EVE_BENCHMARK_WORKSPACE";+export const AGENT_EVAL_DIRECTORY = ".eve-grader";export const AUTHORING_EVAL_DIRECTORY = "/tmp/photon";export const WORLD_EVENTS_PATH = `${AUTHORING_EVAL_DIRECTORY}/world-events.jsonl`;export const SOURCE_ROOT = "/tmp/eve-source";export const SOURCE_ARCHIVE_PATH = "/tmp/eve-source.tar.gz";export const EVE_PACKAGE_PATH = "/tmp/eve-package/eve.tgz";export const AUTHORING_EVAL_DIRECTORY_ENV = "EVE_AUTHORING_EVAL_DIRECTORY";-export const AUTHORING_MODEL = "openai/gpt-5.5";diff --git a/apps/benchmarks/lib/source.mjs b/apps/benchmarks/lib/source.mjsindex 14346a653..33ea86577 100644--- a/apps/benchmarks/lib/source.mjs+++ b/apps/benchmarks/lib/source.mjs@@ -1,6 +1,6 @@import { execFileSync } from "node:child_process";import { createHash } from "node:crypto";-import { mkdtempSync, readFileSync, rmSync } from "node:fs";+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";import { tmpdir } from "node:os";import { join } from "node:path";@@ -45,6 +45,7 @@ function archiveSubject(repositoryRoot, description, label, createArchive, detaidescription,archive,digest,+ dependencyArchive: dependencyArchive(archive),dependencyDigest: dependencyDigest(archive),...details,};@@ -55,7 +56,7 @@ function archiveSubject(repositoryRoot, description, label, createArchive, detaifunction dependencyDigest(archive) {const hash = createHash("sha256");- for (const path of [".npmrc", "package.json", "pnpm-lock.yaml", "pnpm-workspace.yaml"]) {+ for (const path of dependencyPaths(archive)) {const content = execFileSync("tar", ["-xOzf", "-", path], {input: archive,maxBuffer: 10 * 1024 * 1024,@@ -65,6 +66,43 @@ function dependencyDigest(archive) {return hash.digest("hex");}+function dependencyArchive(archive) {+ const directory = mkdtempSync(join(tmpdir(), "eve-authoring-dependencies-"));+ const archivePath = join(directory, "dependencies.tar.gz");+ try {+ for (const path of dependencyPaths(archive)) {+ const destination = join(directory, "input", path);+ mkdirSync(join(destination, ".."), { recursive: true });+ writeFileSync(+ destination,+ execFileSync("tar", ["-xOzf", "-", path], {+ input: archive,+ maxBuffer: 10 * 1024 * 1024,+ }),+ );+ }+ execFileSync("tar", ["-czf", archivePath, "-C", join(directory, "input"), "."]);+ return readFileSync(archivePath);+ } finally {+ rmSync(directory, { recursive: true, force: true });+ }+}++function dependencyPaths(archive) {+ const paths = execFileSync("tar", ["-tzf", "-"], { input: archive, encoding: "utf8" })+ .split("\n")+ .filter(Boolean);+ return paths.filter(+ (path) =>+ path === ".npmrc" ||+ path === "package.json" ||+ path === "pnpm-lock.yaml" ||+ path === "pnpm-workspace.yaml" ||+ /(?:^|\/)package\.json$/u.test(path) ||+ path.startsWith("patches/"),+ );+}+function git(cwd, args, env = process.env) {return execFileSync("git", args, { cwd, env, encoding: "utf8" });}diff --git a/apps/benchmarks/lib/source.test.mjs b/apps/benchmarks/lib/source.test.mjsindex 7d972871e..19c6df031 100644--- a/apps/benchmarks/lib/source.test.mjs+++ b/apps/benchmarks/lib/source.test.mjs@@ -23,11 +23,22 @@ test("archives the working tree without changing the index", () => {assert.equal(existsSync(join(extracted, "ignored.txt")), false);assert.equal(existsSync(join(extracted, "deleted.txt")), false);});+ withExtracted(subject.dependencyArchive, (extracted) => {+ assert.equal(readFileSync(join(extracted, "package.json"), "utf8"), '{"private":true}\n');+ assert.equal(+ readFileSync(join(extracted, "pnpm-lock.yaml"), "utf8"),+ "lockfileVersion: '9.0'\n",+ );+ assert.equal(existsSync(join(extracted, "tracked.txt")), false);+ });assert.equal(git(repository, ["status", "--porcelain"]), statusBefore);writeFileSync(join(repository, "tracked.txt"), "another source change\n");const sourceChange = workingTreeSubject(repository);assert.notEqual(sourceChange.digest, subject.digest);assert.equal(sourceChange.dependencyDigest, subject.dependencyDigest);+ assert.deepEqual(sourceChange.dependencyArchive, subject.dependencyArchive);+ writeFileSync(join(repository, "packages.json"), "not a manifest\n");+ assert.equal(workingTreeSubject(repository).dependencyDigest, subject.dependencyDigest);writeFileSync(join(repository, "pnpm-lock.yaml"), "lockfileVersion: '9.1'\n");assert.notEqual(workingTreeSubject(repository).dependencyDigest, subject.dependencyDigest);} finally {diff --git a/apps/benchmarks/lib/timing.test.mjs b/apps/benchmarks/lib/timing.test.mjsnew file mode 100644index 000000000..adf42aae2--- /dev/null+++ b/apps/benchmarks/lib/timing.test.mjs@@ -0,0 +1,20 @@+import assert from "node:assert/strict";+import { test } from "node:test";++import { BenchmarkTimings } from "./timing.ts";++test("records successful and failed phases", async () => {+ const timings = new BenchmarkTimings();+ await timings.measure("success", async () => undefined, { turns: 1 });+ await assert.rejects(timings.measure("failure", async () => Promise.reject(new Error("nope"))));++ assert.equal(timings.entries.length, 2);+ assert.deepEqual(+ timings.entries.map(({ phase, outcome, details }) => ({ phase, outcome, details })),+ [+ { phase: "success", outcome: "success", details: { turns: 1 } },+ { phase: "failure", outcome: "failure", details: undefined },+ ],+ );+ assert.ok(timings.entries.every((entry) => entry.durationMs >= 0));+});diff --git a/apps/benchmarks/lib/timing.ts b/apps/benchmarks/lib/timing.tsnew file mode 100644index 000000000..15e1c0453--- /dev/null+++ b/apps/benchmarks/lib/timing.ts@@ -0,0 +1,55 @@+export interface BenchmarkTiming {+ readonly phase: string;+ readonly startedAt: string;+ readonly durationMs: number;+ readonly outcome: "success" | "failure";+ readonly details?: Readonly<Record<string, string | number | boolean>>;+}++type TimingDetails = BenchmarkTiming["details"];++export class BenchmarkTimings {+ readonly entries: BenchmarkTiming[] = [];++ async measure<T>(+ phase: string,+ operation: () => Promise<T>,+ details?: TimingDetails,+ ): Promise<T> {+ const startedAt = new Date().toISOString();+ const started = performance.now();+ try {+ const result = await operation();+ this.add(phase, startedAt, performance.now() - started, "success", details);+ return result;+ } catch (error) {+ this.add(phase, startedAt, performance.now() - started, "failure", details);+ throw error;+ }+ }++ record(+ phase: string,+ durationMs: number,+ outcome: BenchmarkTiming["outcome"] = "success",+ details?: TimingDetails,+ ): void {+ this.add(phase, new Date(Date.now() - durationMs).toISOString(), durationMs, outcome, details);+ }++ private add(+ phase: string,+ startedAt: string,+ durationMs: number,+ outcome: BenchmarkTiming["outcome"],+ details: TimingDetails,+ ): void {+ const entry = {+ phase,+ startedAt,+ durationMs: Math.round(durationMs),+ outcome,+ };+ this.entries.push(details === undefined ? entry : { ...entry, details });+ }+}diff --git a/apps/benchmarks/package.json b/apps/benchmarks/package.jsonindex 41c97631f..5775b80ef 100644--- a/apps/benchmarks/package.json+++ b/apps/benchmarks/package.json@@ -8,7 +8,8 @@"benchmark:publish": "node publish.mjs","benchmark:export": "node scripts/export-results.mjs","test": "node --test lib/*.test.mjs lib/setups/*.test.mjs scripts/*.test.mjs",- "typecheck": "tsc --noEmit"+ "typecheck": "tsc --noEmit",+ "benchmark:timings": "node scripts/timings.mjs"},"dependencies": {"@ai-sdk/harness": "1.0.72",diff --git a/apps/benchmarks/publish.mjs b/apps/benchmarks/publish.mjsindex 82ff1ce8e..3d3905863 100644--- a/apps/benchmarks/publish.mjs+++ b/apps/benchmarks/publish.mjs@@ -1,7 +1,6 @@#!/usr/bin/env nodeimport { execFileSync, spawnSync } from "node:child_process";-import { existsSync, mkdirSync, readdirSync, rmSync, writeFileSync } from "node:fs";import { dirname, join, resolve } from "node:path";import { fileURLToPath } from "node:url";import { parseArgs } from "node:util";@@ -13,6 +12,12 @@ import {publishedBenchmarkModels,publishedExperimentId,} from "./lib/benchmark-config.ts";+import {+ prepareFixtures,+ resetExperiments,+ writeExperiment,+ writeSubjectArchives,+} from "./lib/experiment-files.mjs";import { revisionSubject } from "./lib/source.mjs";const appRoot = dirname(fileURLToPath(import.meta.url));@@ -53,7 +58,7 @@ const experimentNames = benchmarks.flatMap((benchmark) =>authoringTreatments.map((treatment) => publishedExperimentId(benchmark, treatment)),);-prepareFixtures();+prepareFixtures(evalsRoot);writeExperiments(subject, revision, benchmarks);console.log(`> eve revision: ${revision}`);@@ -151,47 +156,26 @@ To inspect pending benchmark cells without publishing:return false;}-function prepareFixtures() {- for (const name of fixtureNames()) {- const fixtureRoot = join(evalsRoot, name);- writeFileSync(join(fixtureRoot, "PROMPT.md"), "");- writeFileSync(- join(fixtureRoot, "package.json"),- `${JSON.stringify({ name: `eve-authoring-${name}`, private: true, type: "module" }, null, 2)}\n`,- );- }-}--function fixtureNames() {- return readdirSync(evalsRoot, { withFileTypes: true })- .filter((entry) => entry.isDirectory() && existsSync(join(evalsRoot, entry.name, "CASE.ts")))- .map((entry) => entry.name)- .sort();-}-function writeExperiments(subject, revision, benchmarks) {- rmSync(experimentsRoot, { recursive: true, force: true });- mkdirSync(experimentsRoot, { recursive: true });- const archiveName = `published-${revision.slice(0, 12)}.source.tar.gz`;- writeFileSync(join(experimentsRoot, archiveName), subject.archive);+ resetExperiments(experimentsRoot);+ const { archiveName, dependencyArchiveName } = writeSubjectArchives(+ experimentsRoot,+ subject,+ `published-${revision.slice(0, 12)}`,+ );for (const benchmark of benchmarks) {for (const treatment of authoringTreatments) {- const experimentName = publishedExperimentId(benchmark, treatment);- writeFileSync(- join(experimentsRoot, `${experimentName}.ts`),- `import { readFileSync } from "node:fs";\n` +- `import { authoringExperiment } from "../lib/experiment.js";\n\n` +- `export default authoringExperiment({\n` +- ` archive: readFileSync(new URL(${JSON.stringify(`./${archiveName}`)}, import.meta.url)),\n` +- ` digest: ${JSON.stringify(subject.digest)},\n` +- ` dependencyDigest: ${JSON.stringify(subject.dependencyDigest)},\n` +- ` runs: ${publishedBenchmark.runs},\n` +- ` evals: ${JSON.stringify(publishedBenchmark.caseIds)},\n` +- ` benchmark: ${JSON.stringify(benchmark)},\n` +- ` treatment: ${JSON.stringify(treatment)},\n` +- `});\n`,- );+ writeExperiment(experimentsRoot, publishedExperimentId(benchmark, treatment), {+ archiveName,+ dependencyArchiveName,+ digest: subject.digest,+ dependencyDigest: subject.dependencyDigest,+ runs: publishedBenchmark.runs,+ evals: publishedBenchmark.caseIds,+ benchmark,+ treatment,+ });}}}diff --git a/apps/benchmarks/run.mjs b/apps/benchmarks/run.mjsindex 4ad625743..3f94640af 100644--- a/apps/benchmarks/run.mjs+++ b/apps/benchmarks/run.mjs@@ -1,12 +1,18 @@#!/usr/bin/env nodeimport { spawnSync } from "node:child_process";-import { existsSync, mkdirSync, readdirSync, rmSync, writeFileSync } from "node:fs";+import { existsSync, mkdirSync } from "node:fs";import { dirname, join, resolve } from "node:path";import { fileURLToPath } from "node:url";import { parseArgs } from "node:util";import { findBenchmarkModel, parseAuthoringTreatment } from "./lib/benchmark-config.ts";+import {+ prepareFixtures,+ resetExperiments,+ writeExperiment,+ writeSubjectArchives,+} from "./lib/experiment-files.mjs";import { revisionSubject, workingTreeSubject } from "./lib/source.mjs";const appRoot = dirname(fileURLToPath(import.meta.url));@@ -59,7 +65,7 @@ const subjects =: revisionSubject(repositoryRoot, values.head, "head"),];-prepareFixtures();+prepareFixtures(evalsRoot, selectedEval === undefined ? undefined : [selectedEval]);mkdirSync(join(appRoot, "results"), { recursive: true });writeExperiments(subjects, runs, benchmark, treatment, values.verbose ?? false);const executable = join(appRoot, "node_modules/.bin/agent-eval");@@ -75,44 +81,24 @@ const result = spawnSync(executable, args, {if (result.error) throw result.error;process.exit(result.status ?? 1);-function prepareFixtures() {- const names = selectedEval === undefined ? fixtureNames() : [selectedEval];- for (const name of names) {- const fixtureRoot = join(evalsRoot, name);- writeFileSync(join(fixtureRoot, "PROMPT.md"), "");- writeFileSync(- join(fixtureRoot, "package.json"),- `${JSON.stringify({ name: `eve-authoring-${name}`, private: true, type: "module" }, null, 2)}\n`,- );- }-}--function fixtureNames() {- return readdirSync(evalsRoot, { withFileTypes: true })- .filter((entry) => entry.isDirectory() && existsSync(join(evalsRoot, entry.name, "CASE.ts")))- .map((entry) => entry.name);-}-function writeExperiments(subjects, runs, benchmark, treatment, verbose) {- rmSync(experimentsRoot, { recursive: true, force: true });- mkdirSync(experimentsRoot, { recursive: true });+ resetExperiments(experimentsRoot);for (const subject of subjects) {- const archivePath = join(experimentsRoot, `${subject.label}.source.tar.gz`);- writeFileSync(archivePath, subject.archive);- writeFileSync(- join(experimentsRoot, `${subject.label}.ts`),- `import { readFileSync } from "node:fs";\n` +- `import { authoringExperiment } from "../lib/experiment.js";\n\n` +- `export default authoringExperiment({\n` +- ` archive: readFileSync(new URL(${JSON.stringify(`./${subject.label}.source.tar.gz`)}, import.meta.url)),\n` +- ` digest: ${JSON.stringify(subject.digest)},\n` +- ` dependencyDigest: ${JSON.stringify(subject.dependencyDigest)},\n` +- ` runs: ${runs},\n` +- ` benchmark: ${JSON.stringify(benchmark)},\n` +- ` treatment: ${JSON.stringify(treatment)},\n` +- ` verbose: ${verbose},\n` +- `});\n`,+ const { archiveName, dependencyArchiveName } = writeSubjectArchives(+ experimentsRoot,+ subject,+ subject.label,);+ writeExperiment(experimentsRoot, subject.label, {+ archiveName,+ dependencyArchiveName,+ digest: subject.digest,+ dependencyDigest: subject.dependencyDigest,+ runs,+ benchmark,+ treatment,+ verbose,+ });}}diff --git a/apps/benchmarks/scripts/timings.mjs b/apps/benchmarks/scripts/timings.mjsnew file mode 100644index 000000000..821ddd3d7--- /dev/null+++ b/apps/benchmarks/scripts/timings.mjs@@ -0,0 +1,74 @@+#!/usr/bin/env node++import { existsSync, readFileSync } from "node:fs";+import { dirname, join, resolve } from "node:path";+import { parseArgs } from "node:util";++const { positionals, values } = parseArgs({+ args: process.argv.slice(2),+ allowPositionals: true,+ options: { json: { type: "boolean" }, help: { type: "boolean", short: "h" } },+ strict: true,+});++if (values.help || positionals.length !== 1) {+ console.log("Usage: node scripts/timings.mjs <run-directory-or-timings.json> [--json]");+ process.exit(values.help ? 0 : 1);+}++const timingsPath = resolveTimingsPath(positionals[0]);+const timings = JSON.parse(readFileSync(timingsPath, "utf8"));+if (!Array.isArray(timings)) throw new Error(`${timingsPath} must contain a timing array.`);++if (values.json) {+ console.log(JSON.stringify(timings, null, 2));+ process.exit(0);+}++const phases = timings.filter((timing) => timing.phase !== "run.context");+const width = Math.max(...phases.map((timing) => timing.phase.length));+for (const timing of phases) {+ const details = timing.details === undefined ? "" : ` ${JSON.stringify(timing.details)}`;+ console.log(+ `${timing.phase.padEnd(width)} ${formatDuration(timing.durationMs).padStart(8)} ${timing.outcome}${details}`,+ );+}++const byPhase = new Map(phases.map((timing) => [timing.phase, timing.durationMs]));+const setup = byPhase.get("session.create") ?? 0;+const agent = phases+ .filter((timing) => /^agent\.turn\.\d+$/u.test(timing.phase))+ .reduce((total, timing) => total + timing.durationMs, 0);+const grader = byPhase.get("validation.grader") ?? 0;+const checks = Math.max(+ 0,+ ...phases+ .filter(+ (timing) => timing.phase.startsWith("validation.") && timing.phase !== "validation.grader",+ )+ .map((timing) => timing.durationMs),+);+console.log(`\nSetup: ${formatDuration(setup)}`);+console.log(`Agent: ${formatDuration(agent)}`);+console.log(`Grader: ${formatDuration(grader)}`);+console.log(`Checks: ${formatDuration(checks)}`);+console.log(`Validation: ${formatDuration(grader + checks)}`);+console.log(`Total: ${formatDuration(byPhase.get("run.total") ?? 0)}`);++function resolveTimingsPath(input) {+ const path = resolve(input);+ if (path.endsWith(".json")) return path;+ const candidates = [+ join(path, "project/benchmark/timings.json"),+ join(dirname(path), "project/benchmark/timings.json"),+ ];+ const timingPath = candidates.find(existsSync);+ if (timingPath === undefined) {+ throw new Error(`Could not find project/benchmark/timings.json under ${path}.`);+ }+ return timingPath;+}++function formatDuration(durationMs) {+ return `${(durationMs / 1000).toFixed(1)}s`;+}diff --git a/apps/benchmarks/scripts/timings.test.mjs b/apps/benchmarks/scripts/timings.test.mjsnew file mode 100644index 000000000..e3f413de7--- /dev/null+++ b/apps/benchmarks/scripts/timings.test.mjs@@ -0,0 +1,39 @@+import assert from "node:assert/strict";+import { execFileSync } from "node:child_process";+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { test } from "node:test";++const script = new URL("./timings.mjs", import.meta.url);++test("reports timing phases and aggregate durations", () => {+ const directory = mkdtempSync(join(tmpdir(), "eve-benchmark-timings-"));+ const path = join(directory, "timings.json");+ try {+ writeFileSync(+ path,+ JSON.stringify([+ { phase: "run.context", durationMs: 0, outcome: "success", details: { cache: "cold" } },+ { phase: "dependency-snapshot.get-or-create", durationMs: 1_000, outcome: "success" },+ { phase: "subject-snapshot.get-or-create", durationMs: 2_000, outcome: "success" },+ { phase: "session.create", durationMs: 6_000, outcome: "success" },+ { phase: "agent.turn.1", durationMs: 3_000, outcome: "success" },+ { phase: "validation.grader", durationMs: 6_000, outcome: "success" },+ { phase: "validation.typecheck", durationMs: 2_000, outcome: "success" },+ { phase: "validation.build", durationMs: 4_000, outcome: "success" },+ { phase: "run.total", durationMs: 13_000, outcome: "success" },+ ]),+ );+ const output = execFileSync(process.execPath, [script.pathname, path], { encoding: "utf8" });+ assert.match(output, /dependency-snapshot\.get-or-create\s+1\.0s\s+success/u);+ assert.match(output, /Setup:\s+6\.0s/u);+ assert.match(output, /Agent:\s+3\.0s/u);+ assert.match(output, /Grader:\s+6\.0s/u);+ assert.match(output, /Checks:\s+4\.0s/u);+ assert.match(output, /Validation:\s+10\.0s/u);+ assert.match(output, /Total:\s+13\.0s/u);+ } finally {+ rmSync(directory, { recursive: true, force: true });+ }+});