A local-first multi-agent app development platform, built on PenguinHarness
/**
* The Benchmark API.
*
* - The list reads each benchmark_config.toml's title, description and status (only a literal
* draft locks a Benchmark; draft and failed are themselves, anything else is published),
* lists a Benchmark that never ran but not a directory without a config, and is empty when
* nothing is configured.
* - scoreboard.yaml v2's evaluations pass through: the summary, the Agent each tested, the
* model-written Case and Evaluation averages and the per-case runs; legacy Scoreboard entries
* are neither migrated nor backfilled; the case count is reported.
* - Members read and outsiders get 404; only the owner creates (the server writes the layout
* the Skills read, refusing malformed requests without writing) and deletes a Benchmark whole.
*
* Benchmarks are Project-level, so a new Project arrives with default_agent's sample Benchmark;
* setup deletes that directory (keeping `benchmarks/`), and builtin-agents.test.ts owns its
* assertions. One app for the file; every case works in a Project of its own.
*/
import fs from "node:fs/promises";
import path from "node:path";
import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest";
import { parse as parseToml } from "smol-toml";
import { benchmarksDir } from "@adelie/core";
import type {
BenchmarkCasesResponse,
BenchmarkCreateRequest,
BenchmarkCreateResponse,
BenchmarksResponse,
ProjectCreateResponse,
WorkspaceFilesResponse,
} from "../src/api/types.js";
import { apiClient, canCreateSymlink, createTestApp, provisionUser } from "./helpers.js";
import type { TestApp } from "./helpers.js";
// Symlink creation needs a privilege or Developer Mode on Windows; canCreateSymlink()
// probes once and caches, so these cases still run where the capability exists.
const itWithSymlinks = it.skipIf(!canCreateSymlink());
describe("benchmarks api", () => {
let t: TestApp;
let owner: ReturnType<typeof apiClient>;
let member: ReturnType<typeof apiClient>;
let outsider: ReturnType<typeof apiClient>;
let projectId: string;
let base: string;
beforeAll(async () => {
t = await createTestApp();
const a = await provisionUser(t.app, "owner_a");
const b = await provisionUser(t.app, "member_b");
const c = await provisionUser(t.app, "outsider_c");
owner = apiClient(t.app, a.cookie);
member = apiClient(t.app, b.cookie);
outsider = apiClient(t.app, c.cookie);
});
afterAll(async () => {
await t.cleanup();
});
// Every case works in a Project of its own.
let projects = 0;
beforeEach(async () => {
projects += 1;
const created = (await (
await owner.post("/api/projects", {
projectId: `owner_a-bench_${projects}`,
name: "Bench project",
})
).json()) as ProjectCreateResponse;
projectId = created.project.projectId;
// Creating the Project seeded default_agent's sample Benchmark at the Project level; these
// cases start from an empty benchmarks directory. Only the example goes — the directory
// itself stays, because its absence is what asks for the example to be provisioned again.
await fs.rm(path.join(benchmarksDir(t.root, projectId), "example-benchmark"), {
recursive: true,
force: true,
});
base = `/api/projects/${projectId}/benchmarks`;
expect(
(await owner.post(`/api/projects/${projectId}/members`, { userId: "member_b" })).status,
).toBe(201);
});
it("returns an empty list when unconfigured", async () => {
expect((await (await owner.get(base)).json()) as BenchmarksResponse).toEqual({
benchmarks: [],
});
});
itWithSymlinks(
"current scoreboard: model-written averages, runtime, and runs pass through",
async () => {
const dir = path.join(benchmarksDir(t.root, projectId), "swe-bench-v2");
await fs.mkdir(path.join(dir, "CASE-001-excel-task", "statement"), { recursive: true });
await fs.mkdir(path.join(dir, "CASE-001-excel-task", "statement", "assets"), {
recursive: true,
});
await fs.mkdir(path.join(dir, "CASE-001-excel-task", "rubric"), { recursive: true });
await fs.mkdir(path.join(dir, "CASE-002-web-task", "statement"), { recursive: true });
await fs.mkdir(path.join(dir, "CASE-002-web-task", "rubric"), { recursive: true });
await fs.writeFile(
path.join(dir, "CASE-001-excel-task", "statement", "README.md"),
"# Case 001: Excel cleanup\n\nClean the workbook.",
"utf8",
);
await fs.writeFile(
path.join(dir, "CASE-001-excel-task", "statement", "data.csv"),
"id,value\n1,alpha\n",
"utf8",
);
await fs.writeFile(
path.join(dir, "CASE-001-excel-task", "statement", "assets", "notes.txt"),
"public notes",
"utf8",
);
await fs.writeFile(
path.join(dir, "CASE-001-excel-task", "statement", "large.txt"),
"x".repeat(300 * 1024),
"utf8",
);
await fs.writeFile(
path.join(dir, "CASE-001-excel-task", "rubric", "README.md"),
"# Scoring rubric\n\n- Correct workbook: 100 points",
"utf8",
);
await fs.writeFile(
path.join(dir, "CASE-001-excel-task", "rubric", "expected.json"),
'{"rows": 1}\n',
"utf8",
);
await fs.writeFile(
path.join(dir, "CASE-002-web-task", "statement", "README.md"),
"# Web task\n\nBuild the page.",
"utf8",
);
await fs.writeFile(
path.join(dir, "CASE-002-web-task", "rubric", "README.md"),
"PRIVATE GOLD: never return this text",
"utf8",
);
await fs.symlink(
path.join(dir, "CASE-002-web-task", "rubric", "README.md"),
path.join(dir, "CASE-001-excel-task", "statement", "private-link.md"),
);
await fs.writeFile(
path.join(dir, "benchmark_config.toml"),
`title = "SWE Bench v2"\ndescription = "Example"\nruns = 2\n`,
"utf8",
);
await fs.writeFile(
path.join(dir, "scoreboard.yaml"),
[
"evaluations:",
' - time: "2026-07-16T10:00:00Z"',
' agent_id: "default_agent"',
" version: 3",
' provider: "deepseek"',
' model_id: "deepseek-v4-pro"',
' thinking_level: "medium"',
' summary_title: "Added planning steps to the system Prompt"',
' summary: "Each case run twice and averaged; added planning steps."',
" score: 72.35",
" cost: 0.04",
" duration_ms: 42500",
" cases:",
// Stored averages are authoritative even when inconsistent with the raw Runs.
' - case: "CASE-001-excel-task"',
" score: 80.2",
" cost: 0.04",
" duration_ms: 50000",
" runs:",
" - score: 80",
" cost: null",
" duration_ms: 48000",
' session_id: "session-run-1"',
" - score: 82",
" cost: 0.04",
" duration_ms: 52000",
' session_id: "session-run-2"',
// All unknown Run costs produce a model-written null Case cost; the Evaluation ignores it.
' - case: "CASE-002-web-task"',
" score: 64.5",
" cost: null",
" duration_ms: 35000",
" runs:",
" - score: 60",
" cost: null",
" duration_ms: 30000",
' session_id: "session-run-3"',
" - score: 70",
" cost: null",
" duration_ms: 40000",
' session_id: "session-run-4"',
// A second Agent evaluated by the same Benchmark: the two are peers, and one
// Benchmark measures as many Agents as it is pointed at.
' - time: "2026-07-17T10:00:00Z"',
' agent_id: "report_writer"',
" version: 1",
' provider: "deepseek"',
' model_id: "deepseek-v4-pro"',
' thinking_level: "medium"',
" score: 55",
" cost: null",
" duration_ms: 30000",
" cases:",
' - case: "CASE-001-excel-task"',
" score: 55",
" cost: null",
" duration_ms: 30000",
" runs:",
" - score: 55",
" cost: null",
" duration_ms: 30000",
' session_id: "session-run-5"',
// Recorded before evaluations named an Agent: still listed, just unlabelled.
' - time: "2026-07-18T10:00:00Z"',
" version: 1",
' provider: "deepseek"',
' model_id: "deepseek-v4-pro"',
' thinking_level: "medium"',
" score: 60",
" cost: null",
" duration_ms: 31000",
" cases:",
' - case: "CASE-001-excel-task"',
" score: 60",
" cost: null",
" duration_ms: 31000",
" runs:",
" - score: 60",
" cost: null",
" duration_ms: 31000",
' session_id: "session-run-6"',
].join("\n"),
"utf8",
);
const res = (await (await member.get(base)).json()) as BenchmarksResponse;
const bench = res.benchmarks[0]!;
expect(bench).toMatchObject({
id: "swe-bench-v2",
title: "SWE Bench v2",
description: "Example",
runs: 2,
status: "published",
caseCount: 2,
});
// config carries no model reference (the model lives on each evaluation).
expect("modelId" in bench).toBe(false);
expect("provider" in bench).toBe(false);
// The Agents a Benchmark has evaluated come from its scoreboard, in first-seen order; an
// evaluation without an agent_id contributes none.
expect(bench.agentIds).toEqual(["default_agent", "report_writer"]);
expect(bench.evaluations[2]!.agentId).toBeNull();
const evaluation = bench.evaluations[0]!;
// The evaluation entry carries the Agent it tested, this run's model (as a pair) and a
// summary title (curve series / title-body are displayed separately).
expect(evaluation.agentId).toBe("default_agent");
expect(evaluation.provider).toBe("deepseek");
expect(evaluation.modelId).toBe("deepseek-v4-pro");
expect(evaluation.thinkingLevel).toBe("medium");
expect(evaluation.summaryTitle).toBe("Added planning steps to the system Prompt");
expect(evaluation.summary).toBe("Each case run twice and averaged; added planning steps.");
expect(evaluation.score).toBe(72.35);
expect(evaluation.cost).toBe(0.04);
expect(evaluation.durationMs).toBe(42500);
expect("maxScore" in evaluation).toBe(false);
// Per-case metrics trust the file (80.2, not the Runs' arithmetic mean of 81).
const full = evaluation.cases.find((c) => c.case === "CASE-001-excel-task")!;
expect(full.score).toBe(80.2);
expect(full.cost).toBe(0.04);
expect(full.durationMs).toBe(50000);
expect(full.runs).toEqual([
{ score: 80, cost: null, durationMs: 48000, sessionId: "session-run-1" },
{ score: 82, cost: 0.04, durationMs: 52000, sessionId: "session-run-2" },
]);
const partialCost = evaluation.cases.find((c) => c.case === "CASE-002-web-task")!;
expect(partialCost.score).toBe(64.5);
expect(partialCost.cost).toBeNull();
expect(partialCost.durationMs).toBe(35000);
expect(partialCost.runs).toEqual([
{ score: 60, cost: null, durationMs: 30000, sessionId: "session-run-3" },
{ score: 70, cost: null, durationMs: 40000, sessionId: "session-run-4" },
]);
const caseResponse = (await (
await member.get(`${base}/swe-bench-v2/cases`)
).json()) as BenchmarkCasesResponse;
expect(caseResponse).toEqual({
cases: [
{
id: "CASE-001-excel-task",
title: "Excel cleanup",
},
{
id: "CASE-002-web-task",
title: "Web task",
},
],
});
expect(JSON.stringify(caseResponse)).not.toContain("PRIVATE GOLD");
const filesBase = `${base}/swe-bench-v2/cases/CASE-001-excel-task/files`;
const files = (await (await member.get(filesBase)).json()) as WorkspaceFilesResponse;
expect(files.path).toBe("");
expect(files.entries.map((entry) => `${entry.kind}:${entry.name}`)).toEqual([
"dir:assets",
"file:data.csv",
"file:large.txt",
"file:README.md",
]);
expect(JSON.stringify(files)).not.toContain("private-link.md");
expect(
(
await member.get(
`${filesBase}/content?path=${encodeURIComponent("private-link.md")}&preview=1`,
)
).status,
).toBe(400);
const nested = (await (
await member.get(`${filesBase}?path=${encodeURIComponent("assets")}`)
).json()) as WorkspaceFilesResponse;
expect(nested.entries.map((entry) => entry.name)).toEqual(["notes.txt"]);
const statement = await member.get(
`${filesBase}/content?path=${encodeURIComponent("README.md")}&preview=1`,
);
expect(statement.status).toBe(200);
expect(statement.headers.get("content-type")).toContain("markdown");
expect(await statement.text()).toBe("# Case 001: Excel cleanup\n\nClean the workbook.");
const rubricFilesBase = `${base}/swe-bench-v2/cases/CASE-001-excel-task/rubric/files`;
const rubricFiles = (await (
await member.get(rubricFilesBase)
).json()) as WorkspaceFilesResponse;
expect(rubricFiles.entries.map((entry) => `${entry.kind}:${entry.name}`)).toEqual([
"file:expected.json",
"file:README.md",
]);
const rubric = await member.get(
`${rubricFilesBase}/content?path=${encodeURIComponent("README.md")}&preview=1`,
);
expect(rubric.status).toBe(200);
expect(rubric.headers.get("content-type")).toContain("markdown");
expect(await rubric.text()).toContain("Correct workbook: 100 points");
const large = await member.get(
`${filesBase}/content?path=${encodeURIComponent("large.txt")}&preview=1`,
);
expect(large.status).toBe(200);
expect(large.headers.get("x-content-truncated")).toBe("1");
expect((await large.text()).length).toBe(256 * 1024);
const download = await member.get(
`${filesBase}/content?path=${encodeURIComponent("data.csv")}&download=1`,
);
expect(download.headers.get("content-disposition")).toContain("attachment");
expect(await download.text()).toBe("id,value\n1,alpha\n");
expect(
(await member.get(`${filesBase}/content?path=${encodeURIComponent("../rubric/README.md")}`))
.status,
).toBe(400);
expect(
(
await member.get(
`${rubricFilesBase}/content?path=${encodeURIComponent("../statement/README.md")}`,
)
).status,
).toBe(400);
expect((await outsider.get(`${base}/swe-bench-v2/cases`)).status).toBe(404);
expect((await outsider.get(filesBase)).status).toBe(404);
},
);
it("does not migrate or backfill legacy Scoreboard entries", async () => {
const dir = path.join(benchmarksDir(t.root, projectId), "swe-bench-v1");
await fs.mkdir(path.join(dir, "CASE-001-excel-task", "statement"), { recursive: true });
await fs.writeFile(path.join(dir, "benchmark_config.toml"), `title = "SWE Bench v1"\n`, "utf8");
await fs.writeFile(
path.join(dir, "scoreboard.yaml"),
[
"evaluations:",
' - time: "2026-07-16T10:00:00Z"',
" version: 1",
" score: 62.5",
" cost: 1.25",
" duration_ms: 60000",
" cases:",
' - case: "CASE-001-excel-task"',
" score: 30",
" cost: 0.5",
" duration_ms: 20000",
' session_id: "session-abc"',
' - case: ""', // Bad entry: discarded
" score: 1",
" - time: 42", // Bad evaluation: discarded
" score: 1",
].join("\n"),
"utf8",
);
const res = (await (await member.get(base)).json()) as BenchmarksResponse;
expect(res.benchmarks.map((b) => b.id)).toEqual(["swe-bench-v1"]);
const bench = res.benchmarks[0]!;
expect(bench).toMatchObject({ title: "SWE Bench v1", caseCount: 1 });
expect("runs" in bench).toBe(false);
expect(bench.evaluations).toEqual([]);
expect((await outsider.get(base)).status).toBe(404);
});
it("lists a Benchmark that has never run, but not a directory without a config", async () => {
const dir = benchmarksDir(t.root, projectId);
// Deleted mid-evaluation: the run kept writing, so the directory is back with a case and
// a scoreboard but no config. Not a Benchmark — it must not reach the Evaluation Center.
await fs.mkdir(path.join(dir, "half-deleted", "CASE-001-excel-task", "statement"), {
recursive: true,
});
await fs.writeFile(
path.join(dir, "half-deleted", "CASE-001-excel-task", "statement", "README.md"),
"# Case 001: Excel cleanup\n",
"utf8",
);
await fs.writeFile(
path.join(dir, "half-deleted", "scoreboard.yaml"),
"evaluations: []\n",
"utf8",
);
// Created and never evaluated: a config, cases, and no scoreboard at all. Still a
// Benchmark — having no scores yet is not the same as being incomplete.
await fs.mkdir(path.join(dir, "never-run", "CASE-001-report", "statement"), {
recursive: true,
});
await fs.writeFile(
path.join(dir, "never-run", "benchmark_config.toml"),
'title = "Report writing"\nruns = 2\n',
"utf8",
);
const res = (await (await member.get(base)).json()) as BenchmarksResponse;
expect(res.benchmarks.map((b) => b.id)).toEqual(["never-run"]);
expect(res.benchmarks[0]).toMatchObject({
title: "Report writing",
runs: 2,
caseCount: 1,
evaluations: [],
});
});
it("reads status from the config: literal draft and failed are themselves, everything else is published", async () => {
const dir = benchmarksDir(t.root, projectId);
// Still being written by benchmark-design: the Benchmark is not usable yet.
await fs.mkdir(path.join(dir, "draft-bench"), { recursive: true });
await fs.writeFile(
path.join(dir, "draft-bench", "benchmark_config.toml"),
'title = "Draft"\nruns = 1\nstatus = "draft"\n',
"utf8",
);
// Calibration never produced a Pilot result to freeze: the Benchmark is unusable.
await fs.mkdir(path.join(dir, "failed-bench"), { recursive: true });
await fs.writeFile(
path.join(dir, "failed-bench", "benchmark_config.toml"),
'title = "Failed"\nruns = 1\nstatus = "failed"\n',
"utf8",
);
// Written before the field existed: no status at all is not a lock.
await fs.mkdir(path.join(dir, "legacy-bench"), { recursive: true });
await fs.writeFile(
path.join(dir, "legacy-bench", "benchmark_config.toml"),
'title = "Legacy"\nruns = 1\n',
"utf8",
);
// Neither is a value nobody defined.
await fs.mkdir(path.join(dir, "unknown-bench"), { recursive: true });
await fs.writeFile(
path.join(dir, "unknown-bench", "benchmark_config.toml"),
'title = "Unknown"\nruns = 1\nstatus = "someday"\n',
"utf8",
);
const res = (await (await member.get(base)).json()) as BenchmarksResponse;
expect(res.benchmarks.map((b) => [b.id, b.status])).toEqual([
["draft-bench", "draft"],
["failed-bench", "failed"],
["legacy-bench", "published"],
["unknown-bench", "published"],
]);
});
/** A well-formed create request; the tests below vary one field at a time. */
const createBody: BenchmarkCreateRequest = {
id: "report-writing-v1",
title: "Report writing",
description: "Hard cases for the report writer",
runs: 2,
cases: [
{
id: "CASE-001-contradictions",
title: "Contradicting sources",
statement: "Write a report from the two briefs in this Workspace.\n",
rubric: "- 60 pts: names the contradiction.\n- 40 pts: picks the dated source.\n",
},
{
id: "CASE-002-format",
title: "Strict format",
statement: "Follow the template exactly.",
rubric: "- 100 pts: every template section present.",
},
],
};
it("owner creates a Benchmark by hand and the server writes the Skill layout", async () => {
const res = await owner.post(base, createBody);
expect(res.status).toBe(201);
const { benchmark } = (await res.json()) as BenchmarkCreateResponse;
expect(benchmark).toEqual({
id: "report-writing-v1",
title: "Report writing",
description: "Hard cases for the report writer",
runs: 2,
status: "published",
caseCount: 2,
evaluations: [],
// A Benchmark names no Agent of its own: it has evaluated none until it is run.
agentIds: [],
});
const dir = path.join(benchmarksDir(t.root, projectId), "report-writing-v1");
expect(parseToml(await fs.readFile(path.join(dir, "benchmark_config.toml"), "utf8"))).toEqual({
title: "Report writing",
description: "Hard cases for the report writer",
runs: 2,
status: "published",
});
expect(await fs.readFile(path.join(dir, "scoreboard.yaml"), "utf8")).toBe("evaluations: []\n");
// The statement README opens with the title as its heading (what the case list reads
// back); the rubric is written verbatim, trimmed to one trailing newline.
expect(
await fs.readFile(
path.join(dir, "CASE-001-contradictions", "statement", "README.md"),
"utf8",
),
).toBe("# Contradicting sources\n\nWrite a report from the two briefs in this Workspace.\n");
expect(
await fs.readFile(path.join(dir, "CASE-001-contradictions", "rubric", "README.md"), "utf8"),
).toBe("- 60 pts: names the contradiction.\n- 40 pts: picks the dated source.\n");
const list = (await (await member.get(base)).json()) as BenchmarksResponse;
expect(list.benchmarks.map((b) => b.id)).toEqual(["report-writing-v1"]);
const cases = (await (
await member.get(`${base}/report-writing-v1/cases`)
).json()) as BenchmarkCasesResponse;
expect(cases.cases).toEqual([
{ id: "CASE-001-contradictions", title: "Contradicting sources" },
{ id: "CASE-002-format", title: "Strict format" },
]);
// The same id again is a conflict, and the first Benchmark is left as it was.
const again = await owner.post(base, { ...createBody, title: "Overwrite attempt" });
expect(again.status).toBe(409);
expect(((await again.json()) as { error: { code: string } }).error.code).toBe(
"benchmark_exists",
);
expect(
parseToml(await fs.readFile(path.join(dir, "benchmark_config.toml"), "utf8")),
).toMatchObject({ title: "Report writing" });
});
it("rejects malformed create requests and non-owners without writing anything", async () => {
const first = createBody.cases[0]!;
const bad: unknown[] = [
{ ...createBody, id: "../escape" },
{ ...createBody, id: "" },
{ ...createBody, title: " " },
{ ...createBody, runs: 0 },
{ ...createBody, runs: 1.5 },
{ ...createBody, runs: 1001 },
{ ...createBody, cases: [] },
{ ...createBody, cases: [{ ...first, id: "excel-task" }] },
{ ...createBody, cases: [{ ...first, id: "CASE-001/../x" }] },
{ ...createBody, cases: [first, first] },
{ ...createBody, cases: [{ ...first, rubric: "" }] },
];
for (const body of bad) {
expect((await owner.post(base, body)).status, JSON.stringify(body)).toBe(400);
}
expect((await member.post(base, createBody)).status).toBe(403);
expect((await outsider.post(base, createBody)).status).toBe(404);
// A missing Project is a 404 before any validation.
expect((await owner.post("/api/projects/nobody/benchmarks", createBody)).status).toBe(404);
expect(((await (await owner.get(base)).json()) as BenchmarksResponse).benchmarks).toEqual([]);
});
it("owner deletes a Benchmark directory whole; members and outsiders cannot", async () => {
expect((await owner.post(base, createBody)).status).toBe(201);
const dir = path.join(benchmarksDir(t.root, projectId), "report-writing-v1");
expect((await member.delete(`${base}/report-writing-v1`)).status).toBe(403);
expect((await outsider.delete(`${base}/report-writing-v1`)).status).toBe(404);
await expect(fs.access(dir)).resolves.toBeUndefined();
expect((await owner.delete(`${base}/report-writing-v1`)).status).toBe(204);
await expect(fs.access(dir)).rejects.toThrow();
expect(((await (await owner.get(base)).json()) as BenchmarksResponse).benchmarks).toEqual([]);
// Gone, or never there: both are a plain 404.
expect((await owner.delete(`${base}/report-writing-v1`)).status).toBe(404);
expect((await owner.delete(`${base}/never-existed`)).status).toBe(404);
});
});