A local-first multi-agent app development platform, built on PenguinHarness
/**
* Benchmark score reading: walks the Project's `benchmarks/<id>/`, reads
* `benchmark_config.toml` (title, description, per-case run count `runs`, and the build
* `status`: `draft` while the Benchmark is still being written, `failed` when its calibration
* never produced a result to freeze, `published` otherwise) and `scoreboard.yaml`
* (evaluations[], each carrying the Agent it tested, each case its model-written averages and
* a runs array).
* Content is normally created and refined by the benchmark-design Skill; the server also
* writes the same layout for a Benchmark created by hand (`create`) and removes a Benchmark
* directory whole (`remove`), and never touches a scoreboard.
* `benchmark_config.toml` is what makes a directory a Benchmark: `list` skips one without it.
* Files that are there but corrupt degrade gracefully (title falls back to the directory
* name, scores come back empty) rather than throwing.
*
* Case and Evaluation averages are authoritative file values. The server validates
* the current shape but never recomputes aggregates and does not migrate or backfill
* old Scoreboard formats.
* Docs: /docs/self-improvement § "Benchmark storage".
*/
import fs from "node:fs/promises";
import path from "node:path";
import { parse as parseToml, stringify as stringifyToml } from "smol-toml";
import { parse as parseYaml, stringify as stringifyYaml } from "yaml";
import { benchmarksDir } from "@adelie/core";
import type {
BenchmarkCaseScore,
BenchmarkCaseSummary,
BenchmarkCasesResponse,
BenchmarkEvaluation,
BenchmarkRunScore,
BenchmarkStatus,
BenchmarkSummary,
BenchmarksResponse,
CaseMaterial,
WorkspaceFilesResponse,
} from "../api/types.js";
import type { WorkspaceFileContent, WorkspaceFileReadOptions } from "./workspace-files-service.js";
import { HttpError } from "../http/errors.js";
import { Component, Use } from "@adelie/core/kernel";
import type { Paths } from "../hmr/capabilities.js";
import type { Benchmarks } from "../mechanisms/agents.js";
import type { WorkspaceFiles } from "../mechanisms/workspace.js";
const STATEMENT_TITLE_READ_BYTES = 64 * 1024;
/** One case of a hand-made Benchmark; ids are validated by the route before they reach the filesystem. */
export interface BenchmarkCaseInput {
id: string;
title: string;
statement: string;
rubric: string;
}
export interface BenchmarkCreateInput {
id: string;
title: string;
description?: string;
runs: number;
cases: BenchmarkCaseInput[];
}
function asRecord(v: unknown): Record<string, unknown> {
return v !== null && typeof v === "object" && !Array.isArray(v)
? (v as Record<string, unknown>)
: {};
}
function numberOr(v: unknown): number | undefined {
return typeof v === "number" && Number.isFinite(v) ? v : undefined;
}
function scoreOr(v: unknown): number | undefined {
const value = numberOr(v);
return value !== undefined && value >= 0 && value <= 100 ? value : undefined;
}
function nonNegativeOr(v: unknown): number | undefined {
const value = numberOr(v);
return value !== undefined && value >= 0 ? value : undefined;
}
function nonNegativeIntegerOr(v: unknown): number | undefined {
const value = nonNegativeOr(v);
return value !== undefined && Number.isInteger(value) ? value : undefined;
}
/** `null` is the one valid unknown-cost representation; undefined means invalid input. */
function nullableCostOr(v: unknown): number | null | undefined {
if (v === null) return null;
return nonNegativeOr(v);
}
function stringOr(v: unknown): string | undefined {
return typeof v === "string" && v !== "" ? v : undefined;
}
/**
* The Agent under test on one evaluation. Unlike the runtime fields it never invalidates a
* record: a scoreboard written before evaluations carried one still displays, unlabelled.
*/
function agentIdOr(v: unknown): string | null {
const value = typeof v === "string" ? v.trim() : "";
return value !== "" ? value : null;
}
function isWithin(parent: string, child: string): boolean {
const relative = path.relative(parent, child);
return relative !== "" && !relative.startsWith("..") && !path.isAbsolute(relative);
}
function statementTitle(statement: string, fallback: string): string {
const heading = /^#\s+(.+)$/m.exec(statement)?.[1]?.trim();
return heading?.replace(/^Case\s+\d+\s*:\s*/i, "") || fallback;
}
async function readStatementTitle(readme: string, fallback: string): Promise<string> {
const handle = await fs.open(readme, "r");
try {
const buffer = Buffer.alloc(STATEMENT_TITLE_READ_BYTES);
const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0);
return statementTitle(buffer.subarray(0, bytesRead).toString("utf8"), fallback);
} finally {
await handle.close();
}
}
/** Shapes one current-format Run; a malformed entry invalidates its containing Case. */
function toRun(v: unknown): BenchmarkRunScore | null {
const r = asRecord(v);
const score = scoreOr(r.score);
const cost = nullableCostOr(r.cost);
const durationMs = nonNegativeIntegerOr(r.duration_ms);
const sessionId = stringOr(r.session_id);
if (score === undefined || cost === undefined || durationMs === undefined || !sessionId)
return null;
return { score, cost, durationMs, sessionId };
}
/**
* Shapes one current-format Case. Its stored aggregates are trusted as written:
* this parser intentionally performs no average or consistency calculation.
*/
function toCase(v: unknown): BenchmarkCaseScore | null {
const cr = asRecord(v);
const caseId = stringOr(cr.case);
const score = scoreOr(cr.score);
const cost = nullableCostOr(cr.cost);
const durationMs = nonNegativeIntegerOr(cr.duration_ms);
if (
!caseId ||
score === undefined ||
cost === undefined ||
durationMs === undefined ||
"max_score" in cr ||
!Array.isArray(cr.runs) ||
cr.runs.length === 0
) {
return null;
}
const parsedRuns = cr.runs.map(toRun);
if (parsedRuns.some((run) => run === null)) return null;
const runs = parsedRuns as BenchmarkRunScore[];
return {
case: caseId,
score,
cost,
durationMs,
runs,
};
}
/** Shapes one current-format Evaluation and trusts its stored aggregate metrics. */
function toEvaluation(v: unknown): BenchmarkEvaluation | null {
const r = asRecord(v);
const time = r.time instanceof Date ? r.time.toISOString() : r.time;
const agentId = agentIdOr(r.agent_id);
const score = scoreOr(r.score);
const cost = nullableCostOr(r.cost);
const durationMs = nonNegativeIntegerOr(r.duration_ms);
const summary = stringOr(r.summary);
// Title and body are separate: summary_title is a one-line
// conclusion, summary is the body text.
const summaryTitle = stringOr(r.summary_title);
const modelId = stringOr(r.model_id);
const provider = stringOr(r.provider);
const thinkingLevel = stringOr(r.thinking_level);
const version = nonNegativeIntegerOr(r.version);
if (
typeof time !== "string" ||
time === "" ||
score === undefined ||
cost === undefined ||
durationMs === undefined ||
!modelId ||
!provider ||
!thinkingLevel ||
version === undefined ||
version < 1 ||
!Array.isArray(r.cases) ||
r.cases.length === 0
) {
return null;
}
const parsedCases = r.cases.map(toCase);
if (parsedCases.some((item) => item === null)) return null;
const cases = parsedCases as BenchmarkCaseScore[];
return {
time,
agentId,
...(summaryTitle !== undefined ? { summaryTitle } : {}),
...(summary !== undefined ? { summary } : {}),
modelId,
provider,
thinkingLevel,
score,
version,
cost,
durationMs,
cases,
};
}
@Component()
export class BenchmarkService implements Benchmarks {
@Use() private readonly paths!: Paths;
private get root(): string {
return this.paths.root;
}
@Use() private readonly workspaceFiles!: WorkspaceFiles;
async list(projectId: string): Promise<BenchmarksResponse> {
const dir = benchmarksDir(this.root, projectId);
let items: Array<{ name: string; isDir: boolean }>;
try {
const entries = await fs.readdir(dir, { withFileTypes: true });
items = entries.map((e) => ({ name: e.name, isDir: e.isDirectory() }));
} catch {
return { benchmarks: [] }; // Doesn't exist when unconfigured.
}
const benchmarks: BenchmarkSummary[] = [];
for (const item of items.filter((i) => i.isDir).sort((a, b) => a.name.localeCompare(b.name))) {
const benchDir = path.join(dir, item.name);
// Only `benchmark_config.toml` makes a directory a Benchmark — it is the file the
// evaluation Skills require, and without it there is no title and no run count. A
// Benchmark deleted while an evaluation is still running comes back as the paths that
// run keeps writing, config not among them; that debris is not a Benchmark and is not
// listed. Absence of results is not absence of a Benchmark: one that has never run has
// its config and lists as usual.
try {
await fs.access(path.join(benchDir, "benchmark_config.toml"));
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
continue;
}
benchmarks.push(await this.readBenchmark(benchDir, item.name));
}
return { benchmarks };
}
/**
* Every id `create` refuses as taken, as names only: each entry under `benchmarks/`. Unlike
* `list`, this includes a directory with no `benchmark_config.toml` (the debris of one deleted
* mid-evaluation), and it reads no config and no scoreboard.
*/
async takenIds(projectId: string): Promise<string[]> {
try {
return await fs.readdir(benchmarksDir(this.root, projectId));
} catch (error) {
if ((error as NodeJS.ErrnoException).code === "ENOENT") return [];
throw error;
}
}
/**
* Creates `benchmarks/<id>/` in the layout the evaluation Skills read: `benchmark_config.toml`
* (title, description, runs, status), `scoreboard.yaml` with an empty evaluations list, and
* per case `statement/README.md` (`# <title>`, then the statement) and `rubric/README.md` (the
* rubric verbatim). An existing directory is a 409, never merged into: a Benchmark's scores stay
* comparable only while its cases are rewritten by nothing but the Skills. A half-written
* directory is removed again when a later write fails.
*/
async create(projectId: string, input: BenchmarkCreateInput): Promise<BenchmarkSummary> {
const dir = benchmarksDir(this.root, projectId);
const benchDir = path.join(dir, input.id);
await fs.mkdir(dir, { recursive: true });
try {
// A non-recursive mkdir is the existence check: it fails atomically on a directory
// that is already there, so two creates of one id cannot both proceed.
await fs.mkdir(benchDir);
} catch (error) {
if ((error as NodeJS.ErrnoException).code === "EEXIST") {
throw new HttpError(409, "benchmark_exists", `Benchmark already exists: ${input.id}`);
}
throw error;
}
try {
const config = {
title: input.title,
...(input.description !== undefined && input.description !== ""
? { description: input.description }
: {}),
runs: input.runs,
// A Benchmark made by hand is complete the moment it is submitted: its cases are
// written and frozen, so nothing is left for a Skill to finish.
status: "published",
};
await fs.writeFile(
path.join(benchDir, "benchmark_config.toml"),
`${stringifyToml(config)}\n`,
"utf8",
);
await fs.writeFile(
path.join(benchDir, "scoreboard.yaml"),
stringifyYaml({ evaluations: [] }),
"utf8",
);
for (const item of input.cases) {
const caseDir = path.join(benchDir, item.id);
await fs.mkdir(path.join(caseDir, "statement"), { recursive: true });
await fs.mkdir(path.join(caseDir, "rubric"), { recursive: true });
await fs.writeFile(
path.join(caseDir, "statement", "README.md"),
`# ${item.title.trim()}\n\n${item.statement.trim()}\n`,
"utf8",
);
await fs.writeFile(
path.join(caseDir, "rubric", "README.md"),
`${item.rubric.trim()}\n`,
"utf8",
);
}
} catch (error) {
await fs.rm(benchDir, { recursive: true, force: true });
throw error;
}
return this.readBenchmark(benchDir, input.id);
}
/**
* Removes `benchmarks/<id>/` whole — cases, config and scoreboard. Only a real directory
* counts as existing: a symlink there is not followed, so nothing outside the Project's own
* benchmarks directory can be deleted through this route.
*/
async remove(projectId: string, benchmarkId: string): Promise<void> {
const benchDir = path.join(benchmarksDir(this.root, projectId), benchmarkId);
let isDirectory = false;
try {
isDirectory = (await fs.lstat(benchDir)).isDirectory();
} catch {
// Missing: reported below as not found.
}
if (!isDirectory) {
throw new HttpError(404, "not_found", `Benchmark does not exist: ${benchmarkId}`);
}
await fs.rm(benchDir, { recursive: true, force: true });
}
async listCases(projectId: string, benchmarkId: string): Promise<BenchmarkCasesResponse> {
const baseDir = benchmarksDir(this.root, projectId);
const benchDir = path.join(baseDir, benchmarkId);
let entries: Array<{ name: string; isDirectory(): boolean }>;
let realBaseDir: string;
let realBenchDir: string;
try {
[entries, realBaseDir, realBenchDir] = await Promise.all([
fs.readdir(benchDir, { withFileTypes: true }),
fs.realpath(baseDir),
fs.realpath(benchDir),
]);
} catch {
return { cases: [] };
}
if (!isWithin(realBaseDir, realBenchDir)) return { cases: [] };
const cases: BenchmarkCaseSummary[] = [];
for (const entry of entries
.filter((item) => item.isDirectory() && item.name.startsWith("CASE-"))
.sort((a, b) => a.name.localeCompare(b.name))) {
const fallback: BenchmarkCaseSummary = { id: entry.name, title: entry.name };
try {
const statementDir = await this.caseMaterialRoot(
projectId,
benchmarkId,
entry.name,
"statement",
);
const realReadme = await fs.realpath(path.join(statementDir, "README.md"));
if (!isWithin(statementDir, realReadme)) throw new Error("README escapes Statement");
cases.push({
id: entry.name,
title: await readStatementTitle(realReadme, entry.name),
});
} catch {
cases.push(fallback);
}
}
return { cases };
}
async listCaseFiles(
projectId: string,
benchmarkId: string,
caseId: string,
rel: string,
material: CaseMaterial,
): Promise<WorkspaceFilesResponse> {
const materialRoot = await this.caseMaterialRoot(projectId, benchmarkId, caseId, material);
return this.workspaceFiles.list(materialRoot, rel);
}
async readCaseFile(
projectId: string,
benchmarkId: string,
caseId: string,
rel: string,
material: CaseMaterial,
options?: WorkspaceFileReadOptions,
): Promise<WorkspaceFileContent> {
const materialRoot = await this.caseMaterialRoot(projectId, benchmarkId, caseId, material);
return this.workspaceFiles.read(materialRoot, rel, options);
}
private async caseMaterialRoot(
projectId: string,
benchmarkId: string,
caseId: string,
material: CaseMaterial,
): Promise<string> {
const benchDir = path.join(benchmarksDir(this.root, projectId), benchmarkId);
const caseDir = path.join(benchDir, caseId);
const materialRoot = path.join(caseDir, material);
try {
const [realBenchDir, realCaseDir, realMaterialRoot] = await Promise.all([
fs.realpath(benchDir),
fs.realpath(caseDir),
fs.realpath(materialRoot),
]);
if (
!isWithin(realBenchDir, realCaseDir) ||
path.dirname(realMaterialRoot) !== realCaseDir ||
path.basename(realMaterialRoot) !== material
) {
throw new Error("Case material is not canonical");
}
return realMaterialRoot;
} catch {
throw new HttpError(404, "not_found", `Case ${material} does not exist.`);
}
}
private async readBenchmark(benchDir: string, id: string): Promise<BenchmarkSummary> {
// benchmark_config.toml: title, description, per-case run count and build status (falls
// back to defaults if corrupt). The model isn't part of the config — each evaluation
// carries the Model actually used for that run.
let title = id;
let description: string | undefined;
let runs: number | undefined;
let status: BenchmarkStatus = "published";
try {
const config = asRecord(
parseToml(await fs.readFile(path.join(benchDir, "benchmark_config.toml"), "utf8")),
);
if (typeof config.title === "string" && config.title !== "") title = config.title;
if (typeof config.description === "string" && config.description !== "") {
description = config.description;
}
const configRuns = numberOr(config.runs);
if (configRuns !== undefined && Number.isInteger(configRuns) && configRuns >= 1) {
runs = configRuns;
}
// The two states that make a Benchmark unusable are literal: "draft" while it is still
// being built, "failed" when its calibration never produced a result to freeze. A config
// written before this field existed has none, and an unrecognized value is neither, so
// both read as published.
const raw = stringOr(config.status);
status = raw === "draft" ? "draft" : raw === "failed" ? "failed" : "published";
} catch {
// Missing or corrupt: title falls back to the directory name.
}
// scoreboard.yaml: evaluations[] is appended over time; bad entries are dropped one by one.
let evaluations: BenchmarkEvaluation[] = [];
try {
const scoreboard = asRecord(
parseYaml(await fs.readFile(path.join(benchDir, "scoreboard.yaml"), "utf8")),
);
if (Array.isArray(scoreboard.evaluations)) {
evaluations = scoreboard.evaluations
.map(toEvaluation)
.filter((e): e is BenchmarkEvaluation => e !== null);
}
} catch {
// No scores yet.
}
// Case count: number of semantic Case subfolders.
let caseCount = 0;
try {
const entries = await fs.readdir(benchDir, { withFileTypes: true });
caseCount = entries.filter((e) => e.isDirectory() && e.name.startsWith("CASE-")).length;
} catch {
// Stays at 0.
}
return {
id,
title,
...(description !== undefined ? { description } : {}),
...(runs !== undefined ? { runs } : {}),
status,
caseCount,
evaluations,
// Which Agents this Benchmark has evaluated is a fact of its scoreboard, not of its
// config: first-seen order, so the list reads in the order the Agents were tested.
agentIds: [
...new Set(
evaluations
.map((evaluation) => evaluation.agentId)
.filter((agentId): agentId is string => agentId !== null),
),
],
};
}
}