Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

perf: profile authoring benchmark setup (#2219)

OwenKephart085aaee99913
diff --git a/apps/benchmarks/README.md b/apps/benchmarks/README.md
index 2910939e2..0f4ee6bd9 100644
--- a/apps/benchmarks/README.md
+++ b/apps/benchmarks/README.md
@@ -35,15 +35,26 @@ pnpm benchmark author-000-imessage \
```
The runner archives each subject locally and uploads it to the sandbox. Revisions and local-only
-commits do not need to be pushed. Dependency downloads are cached by lockfile, so source-only
-changes reuse the prepared pnpm store. For one eval and one run, `--verbose` streams setup
-phases, assistant text, tool calls, grading, and build progress.
+commits do not need to be pushed. It maintains two persistent snapshot layers: a dependency
+snapshot keyed by package-manager inputs, and a subject snapshot keyed by the source tree,
+starting point, setup IDs, and bootstrap version. Source-only changes reuse the dependency snapshot
+but create a new subject snapshot. For one eval and one run, `--verbose` streams setup phases,
+assistant text, tool calls, grading, and build progress.
Local runs use the `guided` treatment by default, which keeps the `AGENTS.md` and aliases generated
by `eve init`. Pass `--treatment baseline` to remove those files before the coding agent starts.
Results are written under `apps/benchmarks/results/`. Each run includes the transcript,
-grader output, summary, and copied project files. Vercel Sandbox and AI Gateway credentials are
+grader output, summary, copied project files, and `project/benchmark/timings.json`. The timing
+artifact records the snapshot-cache outcome, source installation and build phases, workspace setup,
+each user turn with token and tool-call counts, and grading and validation durations. Use it to
+separate sandbox setup time from agent time when comparing runs. Print a compact local report with:
+
+```sh
+pnpm benchmark:timings results/current/<timestamp>/<case>/run-1
+```
+
+Pass `--json` to print the original timing artifact. Vercel Sandbox and AI Gateway credentials are
required.
## Publish canonical results
diff --git a/apps/benchmarks/lib/dependency-sandbox.test.mjs b/apps/benchmarks/lib/dependency-sandbox.test.mjs
deleted file mode 100644
index 89df45999..000000000
--- a/apps/benchmarks/lib/dependency-sandbox.test.mjs
+++ /dev/null
@@ -1,29 +0,0 @@
-import assert from "node:assert/strict";
-import { test } from "node:test";
-
-import { dependencySnapshotId } from "./dependency-snapshot.ts";
-
-test("reuses a dependency sandbox's current snapshot", async () => {
- const snapshotId = await dependencySnapshotId({
- currentSnapshotId: "snapshot-current",
- snapshot: async () => {
- throw new Error("snapshot should not be called");
- },
- });
-
- assert.equal(snapshotId, "snapshot-current");
-});
-
-test("explicitly snapshots a dependency sandbox without a current snapshot", async () => {
- let expiration;
- const snapshotId = await dependencySnapshotId({
- currentSnapshotId: undefined,
- snapshot: async (options) => {
- expiration = options?.expiration;
- return { snapshotId: "snapshot-created" };
- },
- });
-
- assert.equal(snapshotId, "snapshot-created");
- assert.equal(expiration, 0);
-});
diff --git a/apps/benchmarks/lib/dependency-sandbox.ts b/apps/benchmarks/lib/dependency-sandbox.ts
index 88d89ad2d..cb16cf905 100644
--- a/apps/benchmarks/lib/dependency-sandbox.ts
+++ b/apps/benchmarks/lib/dependency-sandbox.ts
@@ -3,16 +3,19 @@ import { createVercelSandbox } from "@ai-sdk/sandbox-vercel";
import { Sandbox } from "@vercel/sandbox";
import { SOURCE_ARCHIVE_PATH, SOURCE_ROOT } from "./paths.js";
+import type { BenchmarkTimings } from "./timing.js";
const dependencySnapshots = new Map<string, Promise<string>>();
const subjectSnapshots = new Map<string, Promise<string>>();
export function createDependencyCachedSandbox(options: {
readonly archive: Uint8Array;
+ readonly dependencyArchive: Uint8Array;
readonly dependencyDigest: string;
readonly ports: ReadonlyArray<number>;
readonly env: Readonly<Record<string, string>>;
readonly log: (message: string) => void;
+ readonly timings: BenchmarkTimings;
}): HarnessV1SandboxProvider {
const sessionProvider = (snapshotId: string) =>
createVercelSandbox({
@@ -29,19 +32,22 @@ export function createDependencyCachedSandbox(options: {
async createSession(request = {}) {
options.log("[setup] preparing dependency cache");
const dependencies = await dependencySnapshot(
- options.archive,
+ options.dependencyArchive,
options.dependencyDigest,
options.log,
+ options.timings,
);
if (request.identity === undefined || request.onFirstCreate === undefined) {
return sessionProvider(dependencies).createSession(request);
}
const subject = await subjectSnapshot(
dependencies,
+ options.archive,
request.identity,
options.env,
request.onFirstCreate,
request.abortSignal,
+ options.timings,
);
return sessionProvider(subject).createSession({
sessionId: request.sessionId,
@@ -50,9 +56,10 @@ export function createDependencyCachedSandbox(options: {
},
async resumeSession(request) {
const dependencies = await dependencySnapshot(
- options.archive,
+ options.dependencyArchive,
options.dependencyDigest,
options.log,
+ options.timings,
);
const provider = sessionProvider(dependencies);
if (provider.resumeSession === undefined) {
@@ -67,11 +74,15 @@ function dependencySnapshot(
archive: Uint8Array,
digest: string,
log: (message: string) => void,
+ timings: BenchmarkTimings,
): Promise<string> {
- const name = `eve-benchmark-dependencies-v4-${digest.slice(0, 24)}`;
+ const name = `eve-benchmark-dependencies-v5-${digest.slice(0, 24)}`;
let snapshot = dependencySnapshots.get(name);
- if (snapshot !== undefined) return snapshot;
- snapshot = createDependencySnapshot(name, archive, log);
+ if (snapshot !== undefined) {
+ timings.record("dependency-snapshot.memory-cache", 0, "success", { name });
+ return snapshot;
+ }
+ snapshot = createDependencySnapshot(name, archive, log, timings);
dependencySnapshots.set(name, snapshot);
return snapshot;
}
@@ -80,49 +91,70 @@ async function createDependencySnapshot(
name: string,
archive: Uint8Array,
log: (message: string) => void,
+ timings: BenchmarkTimings,
): Promise<string> {
let created = false;
- const sandbox = await Sandbox.getOrCreate({
- name,
- runtime: "node24",
- timeout: 15 * 60_000,
- persistent: true,
- snapshotExpiration: 0,
- networkPolicy: "allow-all",
- async onCreate(current) {
- created = true;
- log("[setup] fetching workspace dependencies");
- await current.writeFiles([{ path: SOURCE_ARCHIVE_PATH, content: archive }]);
- const command = await current.runCommand("bash", [
- "-lc",
- `mkdir -p ${SOURCE_ROOT} && tar -xzf ${SOURCE_ARCHIVE_PATH} -C ${SOURCE_ROOT} && npm install --global [email protected] [email protected] && cd ${SOURCE_ROOT} && pnpm fetch --frozen-lockfile`,
- ]);
- if (command.exitCode !== 0) {
- throw new Error(
- `Dependency setup failed (${command.exitCode}):\n${await command.stdout()}\n${await command.stderr()}`,
- );
- }
- },
- });
- if (!created && sandbox.currentSnapshotId !== undefined) return sandbox.currentSnapshotId;
- return stopWithSnapshot(sandbox, "Dependency");
+ const sandbox = await timings.measure("dependency-snapshot.get-or-create", () =>
+ Sandbox.getOrCreate({
+ name,
+ runtime: "node24",
+ timeout: 15 * 60_000,
+ persistent: true,
+ snapshotExpiration: 0,
+ networkPolicy: "allow-all",
+ async onCreate(current) {
+ created = true;
+ log("[setup] fetching workspace dependencies");
+ await current.writeFiles([{ path: SOURCE_ARCHIVE_PATH, content: archive }]);
+ const command = await current.runCommand("bash", [
+ "-lc",
+ `mkdir -p ${SOURCE_ROOT} && tar -xzf ${SOURCE_ARCHIVE_PATH} -C ${SOURCE_ROOT} && npm install --global [email protected] [email protected] && cd ${SOURCE_ROOT} && pnpm fetch --frozen-lockfile`,
+ ]);
+ if (command.exitCode !== 0) {
+ throw new Error(
+ `Dependency setup failed (${command.exitCode}):\n${await command.stdout()}\n${await command.stderr()}`,
+ );
+ }
+ },
+ }),
+ );
+ if (!created && sandbox.currentSnapshotId !== undefined) {
+ timings.record("dependency-snapshot.reused", 0, "success", { name });
+ return sandbox.currentSnapshotId;
+ }
+ return timings.measure("dependency-snapshot.publish", () =>
+ stopWithSnapshot(sandbox, "Dependency"),
+ );
}
// Publish bootstrap mutations explicitly; a layered Vercel template can expose
// its inherited snapshot ID before those mutations receive a new snapshot.
function subjectSnapshot(
dependencySnapshotId: string,
+ archive: Uint8Array,
identity: string,
env: Readonly<Record<string, string>>,
bootstrap: NonNullable<
NonNullable<Parameters<HarnessV1SandboxProvider["createSession"]>[0]>["onFirstCreate"]
>,
- abortSignal?: AbortSignal,
+ abortSignal: AbortSignal | undefined,
+ timings: BenchmarkTimings,
): Promise<string> {
const name = `eve-benchmark-subject-${identity}`;
let snapshot = subjectSnapshots.get(name);
- if (snapshot !== undefined) return snapshot;
- snapshot = createSubjectSnapshot(name, dependencySnapshotId, env, bootstrap, abortSignal);
+ if (snapshot !== undefined) {
+ timings.record("subject-snapshot.memory-cache", 0, "success", { name });
+ return snapshot;
+ }
+ snapshot = createSubjectSnapshot(
+ name,
+ dependencySnapshotId,
+ archive,
+ env,
+ bootstrap,
+ abortSignal,
+ timings,
+ );
subjectSnapshots.set(name, snapshot);
return snapshot;
}
@@ -130,31 +162,41 @@ function subjectSnapshot(
async function createSubjectSnapshot(
name: string,
dependencySnapshotId: string,
+ archive: Uint8Array,
env: Readonly<Record<string, string>>,
bootstrap: NonNullable<
NonNullable<Parameters<HarnessV1SandboxProvider["createSession"]>[0]>["onFirstCreate"]
>,
- abortSignal?: AbortSignal,
+ abortSignal: AbortSignal | undefined,
+ timings: BenchmarkTimings,
): Promise<string> {
let created = false;
- const sandbox = await Sandbox.getOrCreate({
- name,
- source: { type: "snapshot", snapshotId: dependencySnapshotId },
- timeout: 15 * 60_000,
- env: { ...env },
- persistent: true,
- snapshotExpiration: 0,
- networkPolicy: "allow-all",
- signal: abortSignal,
- async onCreate(current) {
- created = true;
- const provider = createVercelSandbox({ sandbox: current });
- const session = await provider.createSession({ abortSignal });
- await bootstrap(session.restricted(), { abortSignal });
- },
- });
- if (!created && sandbox.currentSnapshotId !== undefined) return sandbox.currentSnapshotId;
- return stopWithSnapshot(sandbox, "Subject", abortSignal);
+ const sandbox = await timings.measure("subject-snapshot.get-or-create", () =>
+ Sandbox.getOrCreate({
+ name,
+ source: { type: "snapshot", snapshotId: dependencySnapshotId },
+ timeout: 15 * 60_000,
+ env: { ...env },
+ persistent: true,
+ snapshotExpiration: 0,
+ networkPolicy: "allow-all",
+ signal: abortSignal,
+ async onCreate(current) {
+ created = true;
+ await current.writeFiles([{ path: SOURCE_ARCHIVE_PATH, content: archive }]);
+ const provider = createVercelSandbox({ sandbox: current });
+ const session = await provider.createSession({ abortSignal });
+ await bootstrap(session.restricted(), { abortSignal });
+ },
+ }),
+ );
+ if (!created && sandbox.currentSnapshotId !== undefined) {
+ timings.record("subject-snapshot.reused", 0, "success", { name });
+ return sandbox.currentSnapshotId;
+ }
+ return timings.measure("subject-snapshot.publish", () =>
+ stopWithSnapshot(sandbox, "Subject", abortSignal),
+ );
}
async function stopWithSnapshot(
diff --git a/apps/benchmarks/lib/dependency-snapshot.ts b/apps/benchmarks/lib/dependency-snapshot.ts
deleted file mode 100644
index 8ae18e473..000000000
--- a/apps/benchmarks/lib/dependency-snapshot.ts
+++ /dev/null
@@ -1,8 +0,0 @@
-import type { Sandbox } from "@vercel/sandbox";
-
-export async function dependencySnapshotId(
- sandbox: Pick<Sandbox, "currentSnapshotId" | "snapshot">,
-): Promise<string> {
- if (sandbox.currentSnapshotId !== undefined) return sandbox.currentSnapshotId;
- return (await sandbox.snapshot({ expiration: 0 })).snapshotId;
-}
diff --git a/apps/benchmarks/lib/experiment-files.mjs b/apps/benchmarks/lib/experiment-files.mjs
new file mode 100644
index 000000000..27c9d9a65
--- /dev/null
+++ b/apps/benchmarks/lib/experiment-files.mjs
@@ -0,0 +1,52 @@
+import { existsSync, mkdirSync, readdirSync, rmSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+
+export function fixtureNames(evalsRoot) {
+ return readdirSync(evalsRoot, { withFileTypes: true })
+ .filter((entry) => entry.isDirectory() && existsSync(join(evalsRoot, entry.name, "CASE.ts")))
+ .map((entry) => entry.name)
+ .sort();
+}
+
+export function prepareFixtures(evalsRoot, names = fixtureNames(evalsRoot)) {
+ for (const name of names) {
+ const fixtureRoot = join(evalsRoot, name);
+ writeFileSync(join(fixtureRoot, "PROMPT.md"), "");
+ writeFileSync(
+ join(fixtureRoot, "package.json"),
+ `${JSON.stringify({ name: `eve-authoring-${name}`, private: true, type: "module" }, null, 2)}\n`,
+ );
+ }
+}
+
+export function resetExperiments(experimentsRoot) {
+ rmSync(experimentsRoot, { recursive: true, force: true });
+ mkdirSync(experimentsRoot, { recursive: true });
+}
+
+export function writeSubjectArchives(experimentsRoot, subject, name) {
+ const archiveName = `${name}.source.tar.gz`;
+ const dependencyArchiveName = `${name}.dependencies.tar.gz`;
+ writeFileSync(join(experimentsRoot, archiveName), subject.archive);
+ writeFileSync(join(experimentsRoot, dependencyArchiveName), subject.dependencyArchive);
+ return { archiveName, dependencyArchiveName };
+}
+
+export function writeExperiment(experimentsRoot, name, options) {
+ writeFileSync(
+ join(experimentsRoot, `${name}.ts`),
+ `import { readFileSync } from "node:fs";\n` +
+ `import { authoringExperiment } from "../lib/experiment.js";\n\n` +
+ `export default authoringExperiment({\n` +
+ ` archive: readFileSync(new URL(${JSON.stringify(`./${options.archiveName}`)}, import.meta.url)),\n` +
+ ` dependencyArchive: readFileSync(new URL(${JSON.stringify(`./${options.dependencyArchiveName}`)}, import.meta.url)),\n` +
+ ` digest: ${JSON.stringify(options.digest)},\n` +
+ ` dependencyDigest: ${JSON.stringify(options.dependencyDigest)},\n` +
+ ` runs: ${options.runs},\n` +
+ (options.evals === undefined ? "" : ` evals: ${JSON.stringify(options.evals)},\n`) +
+ ` benchmark: ${JSON.stringify(options.benchmark)},\n` +
+ ` treatment: ${JSON.stringify(options.treatment)},\n` +
+ (options.verbose === undefined ? "" : ` verbose: ${options.verbose},\n`) +
+ `});\n`,
+ );
+}
diff --git a/apps/benchmarks/lib/experiment-files.test.mjs b/apps/benchmarks/lib/experiment-files.test.mjs
new file mode 100644
index 000000000..b122d3733
--- /dev/null
+++ b/apps/benchmarks/lib/experiment-files.test.mjs
@@ -0,0 +1,77 @@
+import assert from "node:assert/strict";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { test } from "node:test";
+
+import {
+ fixtureNames,
+ prepareFixtures,
+ resetExperiments,
+ writeExperiment,
+ writeSubjectArchives,
+} from "./experiment-files.mjs";
+
+const subject = {
+ archive: Buffer.from("source"),
+ dependencyArchive: Buffer.from("dependencies"),
+ digest: "source-digest",
+ dependencyDigest: "dependency-digest",
+};
+const benchmark = {
+ id: "test",
+ model: "test/model",
+ displayName: "Test",
+ harness: "OpenCode",
+ support: "supported",
+};
+
+test("materializes fixtures and complete experiment inputs", () => {
+ const root = mkdtempSync(join(tmpdir(), "eve-benchmark-experiments-"));
+ const evals = join(root, "evals");
+ const experiments = join(root, "experiments");
+ try {
+ writeCase(evals, "author-002-second");
+ writeCase(evals, "author-001-first");
+ writeFileSync(join(evals, "not-a-case"), "ignored");
+ assert.deepEqual(fixtureNames(evals), ["author-001-first", "author-002-second"]);
+
+ prepareFixtures(evals);
+ assert.equal(readFileSync(join(evals, "author-001-first", "PROMPT.md"), "utf8"), "");
+ assert.deepEqual(JSON.parse(readFileSync(join(evals, "author-001-first", "package.json"))), {
+ name: "eve-authoring-author-001-first",
+ private: true,
+ type: "module",
+ });
+
+ resetExperiments(experiments);
+ const archives = writeSubjectArchives(experiments, subject, "published-deadbeef");
+ writeExperiment(experiments, "test-opencode--guided", {
+ ...archives,
+ digest: subject.digest,
+ dependencyDigest: subject.dependencyDigest,
+ runs: 3,
+ evals: ["author-001-first"],
+ benchmark,
+ treatment: "guided",
+ });
+
+ assert.equal(readFileSync(join(experiments, archives.archiveName), "utf8"), "source");
+ assert.equal(
+ readFileSync(join(experiments, archives.dependencyArchiveName), "utf8"),
+ "dependencies",
+ );
+ const experiment = readFileSync(join(experiments, "test-opencode--guided.ts"), "utf8");
+ assert.match(experiment, /dependencyArchive: readFileSync/u);
+ assert.match(experiment, /published-deadbeef\.dependencies\.tar\.gz/u);
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+});
+
+function writeCase(evals, name) {
+ const root = join(evals, name);
+ mkdirSync(root, { recursive: true });
+ writeFileSync(join(root, "CASE.ts"), "export default {};\n");
+ assert.ok(existsSync(root));
+}
diff --git a/apps/benchmarks/lib/experiment.ts b/apps/benchmarks/lib/experiment.ts
index 3dc6094cf..f447286ac 100644
--- a/apps/benchmarks/lib/experiment.ts
+++ b/apps/benchmarks/lib/experiment.ts
@@ -6,6 +6,7 @@ import { createAuthoringAgent } from "./harness-agent.js";
export function authoringExperiment(options: {
readonly archive: Uint8Array;
+ readonly dependencyArchive: Uint8Array;
readonly digest: string;
readonly dependencyDigest: string;
readonly runs?: number;
@@ -17,6 +18,7 @@ export function authoringExperiment(options: {
const agent = createAuthoringAgent({
model: options.benchmark.model,
archive: options.archive,
+ dependencyArchive: options.dependencyArchive,
digest: options.digest,
dependencyDigest: options.dependencyDigest,
});
diff --git a/apps/benchmarks/lib/harness-agent.ts b/apps/benchmarks/lib/harness-agent.ts
index 9fed52f71..8ba587c0d 100644
--- a/apps/benchmarks/lib/harness-agent.ts
+++ b/apps/benchmarks/lib/harness-agent.ts
@@ -23,12 +23,14 @@ import {
WORKSPACE,
} from "./paths.js";
import type { AuthoringTokenUsage, AuthoringTranscriptEntry } from "./protocol.js";
+import { BenchmarkTimings } from "./timing.js";
const HARNESS_BRIDGE_PORT = 4172;
-const BOOTSTRAP_VERSION = "v5";
+const BOOTSTRAP_VERSION = "v6";
export function createAuthoringAgent(subject: {
readonly model: string;
readonly archive: Uint8Array;
+ readonly dependencyArchive: Uint8Array;
readonly digest: string;
readonly dependencyDigest: string;
}): Agent {
@@ -57,8 +59,18 @@ export function createAuthoringAgent(subject: {
const setups = [authoringCase.startingPoint.setup, authoringCase.setup].filter(
(setup): setup is AuthoringSetup => setup !== undefined,
);
+ const timings = new BenchmarkTimings();
+ timings.record("run.context", 0, "success", {
+ sourceArchiveBytes: subject.archive.length,
+ dependencyArchiveBytes: subject.dependencyArchive.length,
+ sourceDigest: subject.digest,
+ dependencyDigest: subject.dependencyDigest,
+ startingPoint: authoringCase.startingPoint.id,
+ setupCount: setups.length,
+ });
const sandbox = createDependencyCachedSandbox({
archive: subject.archive,
+ dependencyArchive: subject.dependencyArchive,
dependencyDigest: subject.dependencyDigest,
ports: [HARNESS_BRIDGE_PORT, ...new Set(setups.flatMap((setup) => setup.ports ?? []))],
env: {
@@ -67,6 +79,7 @@ export function createAuthoringAgent(subject: {
...Object.assign({}, ...setups.map((setup) => setup.environment ?? {})),
},
log,
+ timings,
});
const startedAt = Date.now();
const commands: string[] = [];
@@ -89,22 +102,27 @@ export function createAuthoringAgent(subject: {
onBootstrap: async ({ session: bootstrap, workDir }) => {
const bootstrapSandbox = bootstrap as HarnessV1NetworkSandboxSession;
const context = setupContext(bootstrapSandbox, workDir);
- for (const setup of setups) await setup.onBootstrap?.(context);
+ await timings.measure("subject.case-bootstrap", async () => {
+ for (const setup of setups) await setup.onBootstrap?.(context);
+ });
log("[setup] building the selected eve source");
await bootstrapSubject(
bootstrapSandbox,
workDir,
authoringCase.startingPoint.workspace,
subject.archive,
+ timings,
);
},
onSession: async ({ session: current, sessionWorkDir }) => {
activeSandbox = current as HarnessV1NetworkSandboxSession;
workspace = sessionWorkDir;
const context = setupContext(activeSandbox, workspace);
- for (const setup of setups) await setup.onSession?.(context);
+ await timings.measure("session.setup", async () => {
+ for (const setup of setups) await setup.onSession?.(context);
+ if (options.agentOptions?.agentsMd !== true) await installBaselineEveWrapper(context);
+ });
if (options.agentOptions?.agentsMd !== true) {
- await installBaselineEveWrapper(context);
await context.run("rm -f AGENTS.md CLAUDE.md GEMINI.md");
}
},
@@ -113,7 +131,7 @@ export function createAuthoringAgent(subject: {
});
try {
- session = await withBootstrapInitialization(bootstrapHash(authoringCase, subject), () =>
+ session = await timings.measure("session.create", () =>
agent.createSession({ abortSignal: options.signal }),
);
if (activeSandbox === undefined || workspace === undefined) {
@@ -126,22 +144,25 @@ export function createAuthoringAgent(subject: {
send: async (prompt) => {
transcript.push({ role: "user", content: prompt });
if (verbose) console.log(`[user] ${prompt}`);
- const result = verbose
- ? await streamTurn(agent, session!, prompt, options.timeout, options.signal)
- : await agent.generate({
- session: session!,
- prompt,
- timeout: options.timeout,
- abortSignal: options.signal,
- });
- transcript.push({
- role: "assistant",
- content: result.text,
- toolCalls: authoringToolCalls(result.toolCalls),
- usage: normalizeUsage(result.usage),
+ const turn = transcript.filter((entry) => entry.role === "user").length;
+ const result = await timings.measure(`agent.turn.${turn}`, () =>
+ verbose
+ ? streamTurn(agent, session!, prompt, options.timeout, options.signal)
+ : generateTurn(agent, session!, prompt, options.timeout, options.signal),
+ );
+ const toolCalls = result.toolCalls;
+ const usage = result.usage;
+ transcript.push({ role: "assistant", content: result.text, toolCalls, usage });
+ timings.record(`agent.turn.${turn}.summary`, 0, "success", {
+ promptCharacters: prompt.length,
+ responseCharacters: result.text.length,
+ toolCalls: toolCalls.length,
+ inputTokens: usage.inputTokens,
+ outputTokens: usage.outputTokens,
+ reasoningTokens: usage.reasoningTokens,
});
commands.push(...shellCommands(result.toolCalls));
- return { text: result.text, toolCalls: authoringToolCalls(result.toolCalls) };
+ return { text: result.text, toolCalls };
},
});
@@ -178,19 +199,19 @@ export function createAuthoringAgent(subject: {
]);
log("[grade] running deterministic assertions");
- const test = await resultOf(
- activeSandbox,
- `ln -s ${workspace}/${AGENT_EVAL_DIRECTORY} .eve-grader && trap 'rm -f .eve-grader' EXIT && vitest run .eve-grader/EVAL.test.ts`,
- projectWorkspace,
+ const test = await timings.measure("validation.grader", () =>
+ resultOf(
+ activeSandbox!,
+ `ln -s ${workspace}/${AGENT_EVAL_DIRECTORY} .eve-grader && trap 'rm -f .eve-grader' EXIT && vitest run .eve-grader/EVAL.test.ts`,
+ projectWorkspace,
+ ),
);
const scriptsResults = Object.fromEntries(
await Promise.all(
(options.scripts ?? []).map(async (script) => {
log(`[${script}] running`);
- const result = await resultOf(
- activeSandbox!,
- `npm run ${shellQuote(script)}`,
- projectWorkspace,
+ const result = await timings.measure(`validation.${script}`, () =>
+ resultOf(activeSandbox!, `npm run ${shellQuote(script)}`, projectWorkspace),
);
return [
script,
@@ -200,6 +221,12 @@ export function createAuthoringAgent(subject: {
),
);
const scriptsPassed = Object.values(scriptsResults).every((result) => result.success);
+ timings.record("run.total", Date.now() - startedAt);
+ await context.write(
+ `${AGENT_EVAL_DIRECTORY}/timings.json`,
+ JSON.stringify(timings.entries),
+ );
+ logTimingSummary(log, timings);
log(`[result] ${test.exitCode === 0 && scriptsPassed ? "passed" : "failed"}`);
return {
success: test.exitCode === 0 && scriptsPassed,
@@ -211,12 +238,21 @@ export function createAuthoringAgent(subject: {
transcript: harnessTranscript(transcript),
scriptsResults,
sandboxId: activeSandbox.id,
+ generatedFiles: timingArtifact(timings),
};
} catch (error) {
+ timings.record("run.total", Date.now() - startedAt, "failure");
if (activeSandbox !== undefined && workspace !== undefined) {
- await setupContext(activeSandbox, workspace)
- .write(`${AGENT_EVAL_DIRECTORY}/harness-transcript.json`, JSON.stringify(transcript))
- .catch(() => undefined);
+ await Promise.all([
+ setupContext(activeSandbox, workspace).write(
+ `${AGENT_EVAL_DIRECTORY}/harness-transcript.json`,
+ JSON.stringify(transcript),
+ ),
+ setupContext(activeSandbox, workspace).write(
+ `${AGENT_EVAL_DIRECTORY}/timings.json`,
+ JSON.stringify(timings.entries),
+ ),
+ ]).catch(() => undefined);
}
const result: AgentRunResult = {
success: false,
@@ -225,6 +261,7 @@ export function createAuthoringAgent(subject: {
duration: Date.now() - startedAt,
transcript: harnessTranscript(transcript),
scriptsResults: {},
+ generatedFiles: timingArtifact(timings),
};
if (activeSandbox !== undefined) result.sandboxId = activeSandbox.id;
return result;
@@ -235,6 +272,21 @@ export function createAuthoringAgent(subject: {
};
}
+async function generateTurn(
+ agent: HarnessAgent,
+ session: HarnessAgentSession,
+ prompt: string,
+ timeout: number,
+ abortSignal?: AbortSignal,
+): Promise<AuthoringTurn & { usage: AuthoringTokenUsage }> {
+ const result = await agent.generate({ session, prompt, timeout, abortSignal });
+ return {
+ text: result.text,
+ toolCalls: authoringToolCalls(result.toolCalls),
+ usage: normalizeUsage(result.usage),
+ };
+}
+
async function streamTurn(
agent: HarnessAgent,
session: HarnessAgentSession,
@@ -309,22 +361,25 @@ async function bootstrapSubject(
workspace: string,
workspaceKind: "scaffolded" | "empty",
archive: Uint8Array,
+ timings: BenchmarkTimings,
): Promise<void> {
- await sandbox.writeBinaryFile({ path: SOURCE_ARCHIVE_PATH, content: archive });
- await run(
- sandbox,
- [
- `rm -rf ${SOURCE_ROOT} && mkdir -p ${SOURCE_ROOT}`,
- `tar -xzf ${SOURCE_ARCHIVE_PATH} -C ${SOURCE_ROOT}`,
- `pnpm --dir ${SOURCE_ROOT} install --frozen-lockfile --offline`,
- `pnpm --dir ${SOURCE_ROOT} --filter eve build`,
- "mkdir -p /tmp/eve-package",
- `pnpm --dir ${SOURCE_ROOT}/packages/eve pack --pack-destination /tmp/eve-package`,
- `mv $(find /tmp/eve-package -name '*.tgz' -print -quit) ${EVE_PACKAGE_PATH}`,
- `npm install --global --package-lock=false ${EVE_PACKAGE_PATH}`,
- 'ln -sf "$(npm prefix --global)/bin/eve" /usr/local/bin/eve',
- "command -v eve",
- ].join(" && "),
+ await timings.measure("subject.source-upload", async () => {
+ await sandbox.writeBinaryFile({ path: SOURCE_ARCHIVE_PATH, content: archive });
+ });
+ await timings.measure("subject.source-install", () =>
+ run(
+ sandbox,
+ `rm -rf ${SOURCE_ROOT} && mkdir -p ${SOURCE_ROOT} && tar -xzf ${SOURCE_ARCHIVE_PATH} -C ${SOURCE_ROOT} && pnpm --dir ${SOURCE_ROOT} install --frozen-lockfile --offline`,
+ ),
+ );
+ await timings.measure("subject.eve-build", () =>
+ run(sandbox, `pnpm --dir ${SOURCE_ROOT} --filter eve build`),
+ );
+ await timings.measure("subject.eve-pack-and-cli", () =>
+ run(
+ sandbox,
+ `mkdir -p /tmp/eve-package && pnpm --dir ${SOURCE_ROOT}/packages/eve pack --pack-destination /tmp/eve-package && mv $(find /tmp/eve-package -name '*.tgz' -print -quit) ${EVE_PACKAGE_PATH} && ln -sf ${SOURCE_ROOT}/packages/eve/bin/eve.js /usr/local/bin/eve && command -v eve`,
+ ),
);
const artifactsRoot = `${workspace}/${AGENT_EVAL_DIRECTORY}`;
const workspaceCommands: string[] = [];
@@ -339,7 +394,9 @@ async function bootstrapSubject(
if (workspaceKind === "scaffolded") {
workspaceCommands.push(`test -f ${workspace}/package.json`);
}
- await run(sandbox, workspaceCommands.join(" && "));
+ await timings.measure("subject.workspace-bootstrap", () =>
+ run(sandbox, workspaceCommands.join(" && ")),
+ );
}
function setupContext(
@@ -364,7 +421,7 @@ function setupContext(
async function installBaselineEveWrapper(context: AuthoringSetupContext): Promise<void> {
await context.run(`
-cli_path="$(npm prefix --global)/bin/eve"
+cli_path="/usr/local/bin/eve"
test -x "$cli_path"
real_cli="$cli_path.guided"
if [ ! -e "$real_cli" ]; then mv "$cli_path" "$real_cli"; fi
@@ -379,6 +436,16 @@ chmod +x "$cli_path"
`);
}
+function timingArtifact(timings: BenchmarkTimings): Record<string, string> {
+ return { "benchmark/timings.json": `${JSON.stringify(timings.entries, null, 2)}\n` };
+}
+
+function logTimingSummary(log: (message: string) => void, timings: BenchmarkTimings): void {
+ for (const timing of timings.entries) {
+ log(`[timing] ${timing.phase}: ${timing.durationMs}ms (${timing.outcome})`);
+ }
+}
+
function harnessTranscript(transcript: ReadonlyArray<AuthoringTranscriptEntry>): string {
return transcript
.map((entry) => {
@@ -422,43 +489,6 @@ function bootstrapHash(authoringCase: AuthoringCase, subject: { readonly digest:
return `eve-authoring-${BOOTSTRAP_VERSION}-${subject.digest}-${authoringCase.startingPoint.id}-${setupIds}`;
}
-const bootstrapCoordination = globalThis as typeof globalThis & {
- __eveAuthoringBootstrapLocks?: Map<string, Promise<void>>;
- __eveAuthoringBootstrapsReady?: Set<string>;
-};
-
-async function withBootstrapInitialization<T>(
- key: string,
- operation: () => Promise<T>,
-): Promise<T> {
- const ready = (bootstrapCoordination.__eveAuthoringBootstrapsReady ??= new Set());
- if (ready.has(key)) return operation();
-
- const locks = (bootstrapCoordination.__eveAuthoringBootstrapLocks ??= new Map());
- const previous = locks.get(key) ?? Promise.resolve();
- let release = () => {};
- const current = new Promise<void>((resolve) => {
- release = resolve;
- });
- const tail = previous.then(() => current);
- locks.set(key, tail);
-
- await previous;
- if (ready.has(key)) {
- release();
- if (locks.get(key) === tail) locks.delete(key);
- return operation();
- }
- try {
- const result = await operation();
- ready.add(key);
- return result;
- } finally {
- release();
- if (locks.get(key) === tail) locks.delete(key);
- }
-}
-
async function resultOf(
sandbox: HarnessV1NetworkSandboxSession,
command: string,
diff --git a/apps/benchmarks/lib/load-authoring-case.test.mjs b/apps/benchmarks/lib/load-authoring-case.test.mjs
index 4d9659e13..467e2b18a 100644
--- a/apps/benchmarks/lib/load-authoring-case.test.mjs
+++ b/apps/benchmarks/lib/load-authoring-case.test.mjs
@@ -1,5 +1,5 @@
import assert from "node:assert/strict";
-import { readdir } from "node:fs/promises";
+import { readdir, stat } from "node:fs/promises";
import { dirname, resolve } from "node:path";
import test from "node:test";
import { fileURLToPath } from "node:url";
@@ -12,7 +12,7 @@ const jiti = createJiti(import.meta.url, { interopDefault: true, moduleCache: fa
const { loadAuthoringCase } = await jiti.import(resolve(libRoot, "load-authoring-case.ts"));
for (const entry of await readdir(evalsRoot, { withFileTypes: true })) {
- if (!entry.isDirectory()) continue;
+ if (!entry.isDirectory() || !(await hasCase(entry.name))) continue;
test(`loads ${entry.name}/CASE.ts`, async () => {
const authoringCase = await loadAuthoringCase(resolve(evalsRoot, entry.name));
@@ -21,6 +21,14 @@ for (const entry of await readdir(evalsRoot, { withFileTypes: true })) {
});
}
+async function hasCase(name) {
+ try {
+ return (await stat(resolve(evalsRoot, name, "CASE.ts"))).isFile();
+ } catch {
+ return false;
+ }
+}
+
test("loads the named project output directory", async () => {
const authoringCase = await loadAuthoringCase(resolve(evalsRoot, "author-002-new-project"));
assert.equal(authoringCase.projectDirectory, "wayfinder");
diff --git a/apps/benchmarks/lib/paths.ts b/apps/benchmarks/lib/paths.ts
index d1455fea8..f14c9ff80 100644
--- a/apps/benchmarks/lib/paths.ts
+++ b/apps/benchmarks/lib/paths.ts
@@ -1,10 +1,8 @@
export const WORKSPACE = "workspace";
-export const AGENT_EVAL_DIRECTORY = "/tmp/eve-grader";
-export const WORKSPACE_ENV = "EVE_BENCHMARK_WORKSPACE";
+export const AGENT_EVAL_DIRECTORY = ".eve-grader";
export const AUTHORING_EVAL_DIRECTORY = "/tmp/photon";
export const WORLD_EVENTS_PATH = `${AUTHORING_EVAL_DIRECTORY}/world-events.jsonl`;
export const SOURCE_ROOT = "/tmp/eve-source";
export const SOURCE_ARCHIVE_PATH = "/tmp/eve-source.tar.gz";
export const EVE_PACKAGE_PATH = "/tmp/eve-package/eve.tgz";
export const AUTHORING_EVAL_DIRECTORY_ENV = "EVE_AUTHORING_EVAL_DIRECTORY";
-export const AUTHORING_MODEL = "openai/gpt-5.5";
diff --git a/apps/benchmarks/lib/source.mjs b/apps/benchmarks/lib/source.mjs
index 14346a653..33ea86577 100644
--- a/apps/benchmarks/lib/source.mjs
+++ b/apps/benchmarks/lib/source.mjs
@@ -1,6 +1,6 @@
import { execFileSync } from "node:child_process";
import { createHash } from "node:crypto";
-import { mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
@@ -45,6 +45,7 @@ function archiveSubject(repositoryRoot, description, label, createArchive, detai
description,
archive,
digest,
+ dependencyArchive: dependencyArchive(archive),
dependencyDigest: dependencyDigest(archive),
...details,
};
@@ -55,7 +56,7 @@ function archiveSubject(repositoryRoot, description, label, createArchive, detai
function dependencyDigest(archive) {
const hash = createHash("sha256");
- for (const path of [".npmrc", "package.json", "pnpm-lock.yaml", "pnpm-workspace.yaml"]) {
+ for (const path of dependencyPaths(archive)) {
const content = execFileSync("tar", ["-xOzf", "-", path], {
input: archive,
maxBuffer: 10 * 1024 * 1024,
@@ -65,6 +66,43 @@ function dependencyDigest(archive) {
return hash.digest("hex");
}
+function dependencyArchive(archive) {
+ const directory = mkdtempSync(join(tmpdir(), "eve-authoring-dependencies-"));
+ const archivePath = join(directory, "dependencies.tar.gz");
+ try {
+ for (const path of dependencyPaths(archive)) {
+ const destination = join(directory, "input", path);
+ mkdirSync(join(destination, ".."), { recursive: true });
+ writeFileSync(
+ destination,
+ execFileSync("tar", ["-xOzf", "-", path], {
+ input: archive,
+ maxBuffer: 10 * 1024 * 1024,
+ }),
+ );
+ }
+ execFileSync("tar", ["-czf", archivePath, "-C", join(directory, "input"), "."]);
+ return readFileSync(archivePath);
+ } finally {
+ rmSync(directory, { recursive: true, force: true });
+ }
+}
+
+function dependencyPaths(archive) {
+ const paths = execFileSync("tar", ["-tzf", "-"], { input: archive, encoding: "utf8" })
+ .split("\n")
+ .filter(Boolean);
+ return paths.filter(
+ (path) =>
+ path === ".npmrc" ||
+ path === "package.json" ||
+ path === "pnpm-lock.yaml" ||
+ path === "pnpm-workspace.yaml" ||
+ /(?:^|\/)package\.json$/u.test(path) ||
+ path.startsWith("patches/"),
+ );
+}
+
function git(cwd, args, env = process.env) {
return execFileSync("git", args, { cwd, env, encoding: "utf8" });
}
diff --git a/apps/benchmarks/lib/source.test.mjs b/apps/benchmarks/lib/source.test.mjs
index 7d972871e..19c6df031 100644
--- a/apps/benchmarks/lib/source.test.mjs
+++ b/apps/benchmarks/lib/source.test.mjs
@@ -23,11 +23,22 @@ test("archives the working tree without changing the index", () => {
assert.equal(existsSync(join(extracted, "ignored.txt")), false);
assert.equal(existsSync(join(extracted, "deleted.txt")), false);
});
+ withExtracted(subject.dependencyArchive, (extracted) => {
+ assert.equal(readFileSync(join(extracted, "package.json"), "utf8"), '{"private":true}\n');
+ assert.equal(
+ readFileSync(join(extracted, "pnpm-lock.yaml"), "utf8"),
+ "lockfileVersion: '9.0'\n",
+ );
+ assert.equal(existsSync(join(extracted, "tracked.txt")), false);
+ });
assert.equal(git(repository, ["status", "--porcelain"]), statusBefore);
writeFileSync(join(repository, "tracked.txt"), "another source change\n");
const sourceChange = workingTreeSubject(repository);
assert.notEqual(sourceChange.digest, subject.digest);
assert.equal(sourceChange.dependencyDigest, subject.dependencyDigest);
+ assert.deepEqual(sourceChange.dependencyArchive, subject.dependencyArchive);
+ writeFileSync(join(repository, "packages.json"), "not a manifest\n");
+ assert.equal(workingTreeSubject(repository).dependencyDigest, subject.dependencyDigest);
writeFileSync(join(repository, "pnpm-lock.yaml"), "lockfileVersion: '9.1'\n");
assert.notEqual(workingTreeSubject(repository).dependencyDigest, subject.dependencyDigest);
} finally {
diff --git a/apps/benchmarks/lib/timing.test.mjs b/apps/benchmarks/lib/timing.test.mjs
new file mode 100644
index 000000000..adf42aae2
--- /dev/null
+++ b/apps/benchmarks/lib/timing.test.mjs
@@ -0,0 +1,20 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import { BenchmarkTimings } from "./timing.ts";
+
+test("records successful and failed phases", async () => {
+ const timings = new BenchmarkTimings();
+ await timings.measure("success", async () => undefined, { turns: 1 });
+ await assert.rejects(timings.measure("failure", async () => Promise.reject(new Error("nope"))));
+
+ assert.equal(timings.entries.length, 2);
+ assert.deepEqual(
+ timings.entries.map(({ phase, outcome, details }) => ({ phase, outcome, details })),
+ [
+ { phase: "success", outcome: "success", details: { turns: 1 } },
+ { phase: "failure", outcome: "failure", details: undefined },
+ ],
+ );
+ assert.ok(timings.entries.every((entry) => entry.durationMs >= 0));
+});
diff --git a/apps/benchmarks/lib/timing.ts b/apps/benchmarks/lib/timing.ts
new file mode 100644
index 000000000..15e1c0453
--- /dev/null
+++ b/apps/benchmarks/lib/timing.ts
@@ -0,0 +1,55 @@
+export interface BenchmarkTiming {
+ readonly phase: string;
+ readonly startedAt: string;
+ readonly durationMs: number;
+ readonly outcome: "success" | "failure";
+ readonly details?: Readonly<Record<string, string | number | boolean>>;
+}
+
+type TimingDetails = BenchmarkTiming["details"];
+
+export class BenchmarkTimings {
+ readonly entries: BenchmarkTiming[] = [];
+
+ async measure<T>(
+ phase: string,
+ operation: () => Promise<T>,
+ details?: TimingDetails,
+ ): Promise<T> {
+ const startedAt = new Date().toISOString();
+ const started = performance.now();
+ try {
+ const result = await operation();
+ this.add(phase, startedAt, performance.now() - started, "success", details);
+ return result;
+ } catch (error) {
+ this.add(phase, startedAt, performance.now() - started, "failure", details);
+ throw error;
+ }
+ }
+
+ record(
+ phase: string,
+ durationMs: number,
+ outcome: BenchmarkTiming["outcome"] = "success",
+ details?: TimingDetails,
+ ): void {
+ this.add(phase, new Date(Date.now() - durationMs).toISOString(), durationMs, outcome, details);
+ }
+
+ private add(
+ phase: string,
+ startedAt: string,
+ durationMs: number,
+ outcome: BenchmarkTiming["outcome"],
+ details: TimingDetails,
+ ): void {
+ const entry = {
+ phase,
+ startedAt,
+ durationMs: Math.round(durationMs),
+ outcome,
+ };
+ this.entries.push(details === undefined ? entry : { ...entry, details });
+ }
+}
diff --git a/apps/benchmarks/package.json b/apps/benchmarks/package.json
index 41c97631f..5775b80ef 100644
--- a/apps/benchmarks/package.json
+++ b/apps/benchmarks/package.json
@@ -8,7 +8,8 @@
"benchmark:publish": "node publish.mjs",
"benchmark:export": "node scripts/export-results.mjs",
"test": "node --test lib/*.test.mjs lib/setups/*.test.mjs scripts/*.test.mjs",
- "typecheck": "tsc --noEmit"
+ "typecheck": "tsc --noEmit",
+ "benchmark:timings": "node scripts/timings.mjs"
},
"dependencies": {
"@ai-sdk/harness": "1.0.72",
diff --git a/apps/benchmarks/publish.mjs b/apps/benchmarks/publish.mjs
index 82ff1ce8e..3d3905863 100644
--- a/apps/benchmarks/publish.mjs
+++ b/apps/benchmarks/publish.mjs
@@ -1,7 +1,6 @@
#!/usr/bin/env node
import { execFileSync, spawnSync } from "node:child_process";
-import { existsSync, mkdirSync, readdirSync, rmSync, writeFileSync } from "node:fs";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { parseArgs } from "node:util";
@@ -13,6 +12,12 @@ import {
publishedBenchmarkModels,
publishedExperimentId,
} from "./lib/benchmark-config.ts";
+import {
+ prepareFixtures,
+ resetExperiments,
+ writeExperiment,
+ writeSubjectArchives,
+} from "./lib/experiment-files.mjs";
import { revisionSubject } from "./lib/source.mjs";
const appRoot = dirname(fileURLToPath(import.meta.url));
@@ -53,7 +58,7 @@ const experimentNames = benchmarks.flatMap((benchmark) =>
authoringTreatments.map((treatment) => publishedExperimentId(benchmark, treatment)),
);
-prepareFixtures();
+prepareFixtures(evalsRoot);
writeExperiments(subject, revision, benchmarks);
console.log(`> eve revision: ${revision}`);
@@ -151,47 +156,26 @@ To inspect pending benchmark cells without publishing:
return false;
}
-function prepareFixtures() {
- for (const name of fixtureNames()) {
- const fixtureRoot = join(evalsRoot, name);
- writeFileSync(join(fixtureRoot, "PROMPT.md"), "");
- writeFileSync(
- join(fixtureRoot, "package.json"),
- `${JSON.stringify({ name: `eve-authoring-${name}`, private: true, type: "module" }, null, 2)}\n`,
- );
- }
-}
-
-function fixtureNames() {
- return readdirSync(evalsRoot, { withFileTypes: true })
- .filter((entry) => entry.isDirectory() && existsSync(join(evalsRoot, entry.name, "CASE.ts")))
- .map((entry) => entry.name)
- .sort();
-}
-
function writeExperiments(subject, revision, benchmarks) {
- rmSync(experimentsRoot, { recursive: true, force: true });
- mkdirSync(experimentsRoot, { recursive: true });
- const archiveName = `published-${revision.slice(0, 12)}.source.tar.gz`;
- writeFileSync(join(experimentsRoot, archiveName), subject.archive);
+ resetExperiments(experimentsRoot);
+ const { archiveName, dependencyArchiveName } = writeSubjectArchives(
+ experimentsRoot,
+ subject,
+ `published-${revision.slice(0, 12)}`,
+ );
for (const benchmark of benchmarks) {
for (const treatment of authoringTreatments) {
- const experimentName = publishedExperimentId(benchmark, treatment);
- writeFileSync(
- join(experimentsRoot, `${experimentName}.ts`),
- `import { readFileSync } from "node:fs";\n` +
- `import { authoringExperiment } from "../lib/experiment.js";\n\n` +
- `export default authoringExperiment({\n` +
- ` archive: readFileSync(new URL(${JSON.stringify(`./${archiveName}`)}, import.meta.url)),\n` +
- ` digest: ${JSON.stringify(subject.digest)},\n` +
- ` dependencyDigest: ${JSON.stringify(subject.dependencyDigest)},\n` +
- ` runs: ${publishedBenchmark.runs},\n` +
- ` evals: ${JSON.stringify(publishedBenchmark.caseIds)},\n` +
- ` benchmark: ${JSON.stringify(benchmark)},\n` +
- ` treatment: ${JSON.stringify(treatment)},\n` +
- `});\n`,
- );
+ writeExperiment(experimentsRoot, publishedExperimentId(benchmark, treatment), {
+ archiveName,
+ dependencyArchiveName,
+ digest: subject.digest,
+ dependencyDigest: subject.dependencyDigest,
+ runs: publishedBenchmark.runs,
+ evals: publishedBenchmark.caseIds,
+ benchmark,
+ treatment,
+ });
}
}
}
diff --git a/apps/benchmarks/run.mjs b/apps/benchmarks/run.mjs
index 4ad625743..3f94640af 100644
--- a/apps/benchmarks/run.mjs
+++ b/apps/benchmarks/run.mjs
@@ -1,12 +1,18 @@
#!/usr/bin/env node
import { spawnSync } from "node:child_process";
-import { existsSync, mkdirSync, readdirSync, rmSync, writeFileSync } from "node:fs";
+import { existsSync, mkdirSync } from "node:fs";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { parseArgs } from "node:util";
import { findBenchmarkModel, parseAuthoringTreatment } from "./lib/benchmark-config.ts";
+import {
+ prepareFixtures,
+ resetExperiments,
+ writeExperiment,
+ writeSubjectArchives,
+} from "./lib/experiment-files.mjs";
import { revisionSubject, workingTreeSubject } from "./lib/source.mjs";
const appRoot = dirname(fileURLToPath(import.meta.url));
@@ -59,7 +65,7 @@ const subjects =
: revisionSubject(repositoryRoot, values.head, "head"),
];
-prepareFixtures();
+prepareFixtures(evalsRoot, selectedEval === undefined ? undefined : [selectedEval]);
mkdirSync(join(appRoot, "results"), { recursive: true });
writeExperiments(subjects, runs, benchmark, treatment, values.verbose ?? false);
const executable = join(appRoot, "node_modules/.bin/agent-eval");
@@ -75,44 +81,24 @@ const result = spawnSync(executable, args, {
if (result.error) throw result.error;
process.exit(result.status ?? 1);
-function prepareFixtures() {
- const names = selectedEval === undefined ? fixtureNames() : [selectedEval];
- for (const name of names) {
- const fixtureRoot = join(evalsRoot, name);
- writeFileSync(join(fixtureRoot, "PROMPT.md"), "");
- writeFileSync(
- join(fixtureRoot, "package.json"),
- `${JSON.stringify({ name: `eve-authoring-${name}`, private: true, type: "module" }, null, 2)}\n`,
- );
- }
-}
-
-function fixtureNames() {
- return readdirSync(evalsRoot, { withFileTypes: true })
- .filter((entry) => entry.isDirectory() && existsSync(join(evalsRoot, entry.name, "CASE.ts")))
- .map((entry) => entry.name);
-}
-
function writeExperiments(subjects, runs, benchmark, treatment, verbose) {
- rmSync(experimentsRoot, { recursive: true, force: true });
- mkdirSync(experimentsRoot, { recursive: true });
+ resetExperiments(experimentsRoot);
for (const subject of subjects) {
- const archivePath = join(experimentsRoot, `${subject.label}.source.tar.gz`);
- writeFileSync(archivePath, subject.archive);
- writeFileSync(
- join(experimentsRoot, `${subject.label}.ts`),
- `import { readFileSync } from "node:fs";\n` +
- `import { authoringExperiment } from "../lib/experiment.js";\n\n` +
- `export default authoringExperiment({\n` +
- ` archive: readFileSync(new URL(${JSON.stringify(`./${subject.label}.source.tar.gz`)}, import.meta.url)),\n` +
- ` digest: ${JSON.stringify(subject.digest)},\n` +
- ` dependencyDigest: ${JSON.stringify(subject.dependencyDigest)},\n` +
- ` runs: ${runs},\n` +
- ` benchmark: ${JSON.stringify(benchmark)},\n` +
- ` treatment: ${JSON.stringify(treatment)},\n` +
- ` verbose: ${verbose},\n` +
- `});\n`,
+ const { archiveName, dependencyArchiveName } = writeSubjectArchives(
+ experimentsRoot,
+ subject,
+ subject.label,
);
+ writeExperiment(experimentsRoot, subject.label, {
+ archiveName,
+ dependencyArchiveName,
+ digest: subject.digest,
+ dependencyDigest: subject.dependencyDigest,
+ runs,
+ benchmark,
+ treatment,
+ verbose,
+ });
}
}
diff --git a/apps/benchmarks/scripts/timings.mjs b/apps/benchmarks/scripts/timings.mjs
new file mode 100644
index 000000000..821ddd3d7
--- /dev/null
+++ b/apps/benchmarks/scripts/timings.mjs
@@ -0,0 +1,74 @@
+#!/usr/bin/env node
+
+import { existsSync, readFileSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+import { parseArgs } from "node:util";
+
+const { positionals, values } = parseArgs({
+ args: process.argv.slice(2),
+ allowPositionals: true,
+ options: { json: { type: "boolean" }, help: { type: "boolean", short: "h" } },
+ strict: true,
+});
+
+if (values.help || positionals.length !== 1) {
+ console.log("Usage: node scripts/timings.mjs <run-directory-or-timings.json> [--json]");
+ process.exit(values.help ? 0 : 1);
+}
+
+const timingsPath = resolveTimingsPath(positionals[0]);
+const timings = JSON.parse(readFileSync(timingsPath, "utf8"));
+if (!Array.isArray(timings)) throw new Error(`${timingsPath} must contain a timing array.`);
+
+if (values.json) {
+ console.log(JSON.stringify(timings, null, 2));
+ process.exit(0);
+}
+
+const phases = timings.filter((timing) => timing.phase !== "run.context");
+const width = Math.max(...phases.map((timing) => timing.phase.length));
+for (const timing of phases) {
+ const details = timing.details === undefined ? "" : ` ${JSON.stringify(timing.details)}`;
+ console.log(
+ `${timing.phase.padEnd(width)} ${formatDuration(timing.durationMs).padStart(8)} ${timing.outcome}${details}`,
+ );
+}
+
+const byPhase = new Map(phases.map((timing) => [timing.phase, timing.durationMs]));
+const setup = byPhase.get("session.create") ?? 0;
+const agent = phases
+ .filter((timing) => /^agent\.turn\.\d+$/u.test(timing.phase))
+ .reduce((total, timing) => total + timing.durationMs, 0);
+const grader = byPhase.get("validation.grader") ?? 0;
+const checks = Math.max(
+ 0,
+ ...phases
+ .filter(
+ (timing) => timing.phase.startsWith("validation.") && timing.phase !== "validation.grader",
+ )
+ .map((timing) => timing.durationMs),
+);
+console.log(`\nSetup: ${formatDuration(setup)}`);
+console.log(`Agent: ${formatDuration(agent)}`);
+console.log(`Grader: ${formatDuration(grader)}`);
+console.log(`Checks: ${formatDuration(checks)}`);
+console.log(`Validation: ${formatDuration(grader + checks)}`);
+console.log(`Total: ${formatDuration(byPhase.get("run.total") ?? 0)}`);
+
+function resolveTimingsPath(input) {
+ const path = resolve(input);
+ if (path.endsWith(".json")) return path;
+ const candidates = [
+ join(path, "project/benchmark/timings.json"),
+ join(dirname(path), "project/benchmark/timings.json"),
+ ];
+ const timingPath = candidates.find(existsSync);
+ if (timingPath === undefined) {
+ throw new Error(`Could not find project/benchmark/timings.json under ${path}.`);
+ }
+ return timingPath;
+}
+
+function formatDuration(durationMs) {
+ return `${(durationMs / 1000).toFixed(1)}s`;
+}
diff --git a/apps/benchmarks/scripts/timings.test.mjs b/apps/benchmarks/scripts/timings.test.mjs
new file mode 100644
index 000000000..e3f413de7
--- /dev/null
+++ b/apps/benchmarks/scripts/timings.test.mjs
@@ -0,0 +1,39 @@
+import assert from "node:assert/strict";
+import { execFileSync } from "node:child_process";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { test } from "node:test";
+
+const script = new URL("./timings.mjs", import.meta.url);
+
+test("reports timing phases and aggregate durations", () => {
+ const directory = mkdtempSync(join(tmpdir(), "eve-benchmark-timings-"));
+ const path = join(directory, "timings.json");
+ try {
+ writeFileSync(
+ path,
+ JSON.stringify([
+ { phase: "run.context", durationMs: 0, outcome: "success", details: { cache: "cold" } },
+ { phase: "dependency-snapshot.get-or-create", durationMs: 1_000, outcome: "success" },
+ { phase: "subject-snapshot.get-or-create", durationMs: 2_000, outcome: "success" },
+ { phase: "session.create", durationMs: 6_000, outcome: "success" },
+ { phase: "agent.turn.1", durationMs: 3_000, outcome: "success" },
+ { phase: "validation.grader", durationMs: 6_000, outcome: "success" },
+ { phase: "validation.typecheck", durationMs: 2_000, outcome: "success" },
+ { phase: "validation.build", durationMs: 4_000, outcome: "success" },
+ { phase: "run.total", durationMs: 13_000, outcome: "success" },
+ ]),
+ );
+ const output = execFileSync(process.execPath, [script.pathname, path], { encoding: "utf8" });
+ assert.match(output, /dependency-snapshot\.get-or-create\s+1\.0s\s+success/u);
+ assert.match(output, /Setup:\s+6\.0s/u);
+ assert.match(output, /Agent:\s+3\.0s/u);
+ assert.match(output, /Grader:\s+6\.0s/u);
+ assert.match(output, /Checks:\s+4\.0s/u);
+ assert.match(output, /Validation:\s+10\.0s/u);
+ assert.match(output, /Total:\s+13\.0s/u);
+ } finally {
+ rmSync(directory, { recursive: true, force: true });
+ }
+});