The Open Framework for Building Agents
import test from "node:test";
import assert from "node:assert/strict";
import { copyFileSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { publishRun } from "./publish.mjs";
import { METRICS_VERSION, STANDARD_METRICS } from "../eval-metrics/standard.mjs";
import { buildRecords, readLegs } from "./records.mjs";
const SHA = "a".repeat(40);
const plan = {
schema_version: 1,
sha: SHA,
planned_at: "2026-01-01T00:00:00.000Z",
registry: {
attempts: 1,
models: [{ name: "alpha", id: "acme/alpha", release: "2026-09-22" }],
mock_worlds: ["local"],
fixtures: [
{
name: "agent-tools",
dir: "e2e/fixtures/agent-tools",
digest: "f".repeat(64),
eval_ids: ["static-tools/ends-turn-function", "static-tools/no-reply"],
},
],
},
legs: [
{
leg_id: "live__agent-tools__alpha__local__1",
kind: "live",
fixture: "agent-tools",
dir: "e2e/fixtures/agent-tools",
model_name: "alpha",
model_id: "acme/alpha",
world: "local",
world_package: "",
attempt: 1,
},
{
leg_id: "mock__agent-tools__mock__local__1",
kind: "mock",
fixture: "agent-tools",
dir: "e2e/fixtures/agent-tools",
model_name: "mock",
model_id: "mock",
world: "local",
world_package: "",
attempt: 1,
},
],
};
/** One uploaded live leg whose run produced one of its two expected evals; the mock leg is missing. */
function legsDirectory(t, meta = {}) {
const root = mkdtempSync(join(tmpdir(), "eve-benchmark-"));
t.after(() => rmSync(root, { recursive: true, force: true }));
const leg = join(root, "benchmark-leg-live__agent-tools__alpha__local__1");
mkdirSync(join(leg, "run", "evals", "static-tools"), { recursive: true });
writeFileSync(
join(leg, "meta.json"),
JSON.stringify({
leg_id: "live__agent-tools__alpha__local__1",
sha: SHA,
runner_os: "Linux",
...meta,
}),
);
writeEval(leg, "static-tools/ends-turn-function", "acme/alpha");
writeFileSync(join(leg, "run", "summary.json"), "{}");
return root;
}
function writeEval(leg, evalId, modelId) {
const artifact = {
id: evalId,
verdict: "passed",
assertions: [{ name: "replied", severity: "gate", score: 1, passed: true, extra: "x" }],
startedAt: "2026-01-01T00:00:00.000Z",
completedAt: "2026-01-01T00:00:01.500Z",
result: {
runtimeIdentity: { eveVersion: "0.71.2" },
derived: { models: [modelId] },
},
};
writeFileSync(join(leg, "run", "evals", `${evalId}.json`), JSON.stringify(artifact));
}
const build = (legsDir) => buildWithExclusions(legsDir).records;
const buildWithExclusions = (legsDir) =>
buildRecords({
plan,
legs: readLegs(plan, legsDir),
githubRunId: "123",
runAttempt: 1,
});
test("builds records and turns missing legs and evals into missing records", (t) => {
const records = build(legsDirectory(t));
assert.deepEqual(
records.map((r) => [r.model_id, r.eval_id, r.missing, r.artifact_path]),
[
["acme/alpha", "static-tools/ends-turn-function", null, "raw/agent-tools/alpha/local/1"],
["acme/alpha", "static-tools/no-reply", "eval-missing", "raw/agent-tools/alpha/local/1"],
["mock", "static-tools/ends-turn-function", "job-missing", null],
["mock", "static-tools/no-reply", "job-missing", null],
],
);
const [completed] = records;
assert.equal(completed.eve_version, "0.71.2");
assert.equal(completed.model_release, "2026-09-22");
assert.equal(completed.verdict, "passed");
assert.equal(completed.duration_ms, 1500);
assert.deepEqual(completed.assertions, [
{ name: "replied", severity: "gate", score: 1, passed: true },
]);
assert.equal(records[2].model_release, null);
assert.equal(records[2].duration_ms, null);
assert.equal(completed.outcome, "completed");
assert.deepEqual(completed.metrics["latency.turn_ms"], {
status: "unavailable",
reason: "missing-session-capture",
});
});
test("attaches metrics to captured attempts and marks missing attempts as gaps", (t) => {
const legsDir = legsDirectory(t);
// A real captured artifact, with session events, in place of the minimal one.
copyFileSync(
new URL("../eval-metrics/fixtures/agent-tools-ends-turn-function.json", import.meta.url),
join(
legsDir,
"benchmark-leg-live__agent-tools__alpha__local__1",
"run",
"evals",
"static-tools",
"ends-turn-function.json",
),
);
const [captured, missingEval, missingJob] = build(legsDir);
assert.equal(captured.outcome, "completed");
assert.equal(captured.gap_reason, null);
assert.equal(captured.metrics_version, METRICS_VERSION);
assert.deepEqual(Object.keys(captured.metrics).sort(), Object.keys(STANDARD_METRICS).sort());
assert.deepEqual(captured.metrics["tokens.input"], { status: "measured", value: 1117 });
// The fixture's model reports no cost: unavailable with a reason, never zero.
assert.deepEqual(captured.metrics["cost.total_usd"], {
status: "unavailable",
reason: "missing-rolled-up-cost",
});
assert.deepEqual(captured.failures, []);
for (const [record, reason] of [
[missingEval, "eval-missing"],
[missingJob, "job-missing"],
]) {
assert.equal(record.outcome, "gap");
assert.equal(record.gap_reason, reason);
assert.deepEqual(record.metrics, {});
assert.deepEqual(record.failures, []);
}
});
test("excludes live-leg evals whose model steps all ran on another model", (t) => {
const legsDir = legsDirectory(t);
const leg = join(legsDir, "benchmark-leg-live__agent-tools__alpha__local__1");
// Some fixtures script delegation evals with eve's mock models even on live legs.
writeEval(leg, "static-tools/no-reply", "eve-mock/notebook-parent");
const { records, excluded } = buildWithExclusions(legsDir);
assert.deepEqual(excluded, [
{ leg_id: "live__agent-tools__alpha__local__1", eval_id: "static-tools/no-reply" },
]);
const live = records.filter((r) => r.model_id === "acme/alpha");
assert.deepEqual(
live.map((r) => r.eval_id),
["static-tools/ends-turn-function"],
"an excluded eval is neither a result nor missing",
);
});
test("rejects artifacts that were not planned or ran at another commit", (t) => {
assert.throws(
() => readLegs(plan, legsDirectory(t, { sha: "b".repeat(40) })),
/ran at b+, but the plan is for a+/,
);
const unplanned = { ...plan, legs: plan.legs.slice(1) };
assert.throws(() => readLegs(unplanned, legsDirectory(t)), /was not planned/);
});
function memoryStore() {
const objects = new Map();
const writes = [];
return {
objects,
writes,
async put(pathname, body) {
if (objects.has(pathname))
throw new Error(`Vercel Blob: This blob already exists (${pathname}).`);
objects.set(pathname, Buffer.from(body));
writes.push(pathname);
},
async read(pathname) {
return objects.get(pathname) ?? null;
},
};
}
test("publishes raw trees and records before the manifest, idempotently and immutably", async (t) => {
const legsDir = legsDirectory(t);
const legs = readLegs(plan, legsDir);
const recordsJsonl = build(legsDir)
.map((record) => `${JSON.stringify(record)}\n`)
.join("");
const store = memoryStore();
const input = {
plan,
legs,
recordsJsonl,
githubRunId: "123",
runAttempt: 1,
eveVersion: "0.71.2",
store,
};
const result = await publishRun(input);
assert.equal(result.prefix, "runs/123/1");
assert.equal(store.writes.at(-1), "runs/123/1/manifest.json");
assert(
store.objects.has(
"runs/123/1/raw/agent-tools/alpha/local/1/run/evals/static-tools/ends-turn-function.json",
),
);
assert(store.objects.has("runs/123/1/raw/agent-tools/alpha/local/1/meta.json"));
const manifest = JSON.parse(store.objects.get("runs/123/1/manifest.json"));
assert.equal(manifest.record_count, 4);
assert.deepEqual(
manifest.legs.map((leg) => leg.uploaded),
[true, false],
);
const writes = store.writes.length;
await publishRun(input);
assert.equal(store.writes.length, writes, "identical re-run is a no-op");
await assert.rejects(
publishRun({ ...input, recordsJsonl: recordsJsonl.split("\n").slice(1).join("\n") }),
/records.jsonl was already published with different content/,
);
});