A local-first multi-agent app development platform, built on PenguinHarness
/**
* Example Benchmark provisioning tests:
* default_agent initialization pre-seeds the Project-level benchmarks/example-benchmark/ (a
* parseable config, runs=2, a scoreboard with three self-consistent evaluations labelled with
* the Agent they tested); an ordinary Agent's creation seeds nothing; the decision is made on
* the benchmarks/ DIRECTORY, so a Project that already holds Benchmarks of its own gets no
* example, and loading default_agent re-creates one only when that directory is gone.
*/
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { parse as parseToml } from "smol-toml";
import { parse as parseYaml } from "yaml";
import {
DEFAULT_AGENT_ID,
DEFAULT_PROJECT_ID,
EXAMPLE_BENCHMARK_ID,
agentStateDir,
benchmarksDir,
buildExampleScoreboard,
loadAgentState,
provisionProjectAgents,
} from "../src/state/index.js";
let tmpRoot: string;
let prevHome: string | undefined;
beforeEach(async () => {
prevHome = process.env.PENGUIN_HOME;
tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), "penguin-bench-"));
process.env.PENGUIN_HOME = tmpRoot;
});
afterEach(async () => {
if (prevHome === undefined) delete process.env.PENGUIN_HOME;
else process.env.PENGUIN_HOME = prevHome;
await fs.rm(tmpRoot, { recursive: true, force: true });
});
async function exists(p: string): Promise<boolean> {
try {
await fs.access(p);
return true;
} catch {
return false;
}
}
interface RunScore {
score: number;
cost: number | null;
duration_ms: number;
session_id: string;
}
interface CaseScore extends Omit<RunScore, "session_id"> {
case: string;
runs: RunScore[];
}
interface Evaluation extends Omit<CaseScore, "case" | "runs"> {
time: string;
agent_id: string;
version: number;
provider: string;
model_id: string;
thinking_level: string;
summary_title: string;
summary: string;
cases: CaseScore[];
}
describe("example benchmark provisioning", () => {
it("default_agent init creates a parseable example benchmark (config + scoreboard + cases)", async () => {
await loadAgentState({ init: {} });
const dir = path.join(benchmarksDir(tmpRoot, DEFAULT_PROJECT_ID), EXAMPLE_BENCHMARK_ID);
// benchmark_config.toml: title/description/runs=2/status=published; contains no model
// reference (the model is recorded on each evaluation instead).
const config = parseToml(await fs.readFile(path.join(dir, "benchmark_config.toml"), "utf8"));
expect(config.title).toBe("Example Benchmark");
expect(String(config.description)).toContain("built-in example");
expect(String(config.description)).toContain("Replace it with your own");
expect(Number(config.runs)).toBe(2);
expect(config.status).toBe("published");
expect(config).not.toHaveProperty("provider");
expect(config).not.toHaveProperty("model_id");
// Two cases: statement/ and rubric/ each use README.md as the index.
for (const caseId of ["CASE-001-file-summary", "CASE-002-data-cleanup"]) {
const statement = await fs.readFile(path.join(dir, caseId, "statement", "README.md"), "utf8");
const rubric = await fs.readFile(path.join(dir, caseId, "rubric", "README.md"), "utf8");
expect(statement.length).toBeGreaterThan(50);
expect(rubric).toContain("pts");
expect(rubric).toContain("max 100 points");
}
// scoreboard.yaml: 3 evaluations, with version/time increasing and scores rising, and
// 2 runs per case.
const scoreboard = parseYaml(await fs.readFile(path.join(dir, "scoreboard.yaml"), "utf8")) as {
evaluations: Evaluation[];
};
expect(scoreboard.evaluations).toHaveLength(3);
expect(JSON.stringify(scoreboard)).not.toContain("max_score");
expect(scoreboard.evaluations.map((e) => e.version)).toEqual([1, 2, 3]);
const times = scoreboard.evaluations.map((e) => new Date(e.time).getTime());
expect(times[0]!).toBeLessThan(times[1]!);
expect(times[1]!).toBeLessThan(times[2]!);
const scores = scoreboard.evaluations.map((e) => e.score);
expect(scores[0]!).toBeLessThan(scores[1]!);
expect(scores[1]!).toBeLessThan(scores[2]!);
// Each evaluation carries the Agent it tested, the actual model used for that run (as a
// pair) and a summary title; the example consistently evaluates default_agent on
// deepseek-v4-pro.
expect(scoreboard.evaluations.map((e) => e.model_id)).toEqual([
"deepseek-v4-pro",
"deepseek-v4-pro",
"deepseek-v4-pro",
]);
for (const e of scoreboard.evaluations) {
expect(e.agent_id).toBe(DEFAULT_AGENT_ID);
expect(e.provider).toBe("deepseek");
expect(e.thinking_level).toBe("medium");
expect(e.summary_title.length).toBeGreaterThan(0);
expect(e.summary.toLowerCase()).toContain("example");
expect(e.cases).toHaveLength(2);
for (const c of e.cases) {
expect(c.runs).toHaveLength(2);
for (const r of c.runs) {
expect(r.session_id).toMatch(/^session-\d{4}-\d{2}-\d{2}-\d{2}-\d{2}-\d{2}-[0-9a-f]{8}$/);
}
}
}
});
it("scoreboard numbers preserve Run cost precision and use model-written averages", async () => {
const { evaluations } = buildExampleScoreboard();
expect(evaluations[0]?.cases[0]?.runs.map((run) => run.cost)).toEqual([0.012, 0.014]);
const avg = (vals: number[]): number => vals.reduce((a, b) => a + b, 0) / vals.length;
const knownCostAvg = (vals: Array<number | null>): number | null => {
const known = vals.filter((value): value is number => value !== null);
return known.length > 0 ? avg(known) : null;
};
for (const e of evaluations) {
for (const c of e.cases) {
expect(c.runs.every((run) => run.score >= 0 && run.score <= 100)).toBe(true);
expect(c.score).toBe(Math.round(avg(c.runs.map((r) => r.score)) * 100) / 100);
const expectedCost = knownCostAvg(c.runs.map((r) => r.cost));
expect(c.cost).toBe(
expectedCost === null ? null : Math.round(expectedCost * 1_000_000) / 1_000_000,
);
expect(c.duration_ms).toBe(Math.round(avg(c.runs.map((r) => r.duration_ms))));
}
expect(e.score).toBe(Math.round(avg(e.cases.map((c) => c.score)) * 100) / 100);
const expectedCost = knownCostAvg(e.cases.map((c) => c.cost));
expect(e.cost).toBe(
expectedCost === null ? null : Math.round(expectedCost * 1_000_000) / 1_000_000,
);
expect(e.duration_ms).toBe(Math.round(avg(e.cases.map((c) => c.duration_ms))));
}
});
it("provisionProjectAgents seeds the example benchmark at the Project level", async () => {
await provisionProjectAgents({ root: tmpRoot, projectId: "proj_x" });
expect(await exists(path.join(benchmarksDir(tmpRoot, "proj_x"), EXAMPLE_BENCHMARK_ID))).toBe(
true,
);
});
it("does not create benchmarks when a non-default agent is created", async () => {
await loadAgentState({ init: {}, agentId: "worker" });
expect(await exists(benchmarksDir(tmpRoot, DEFAULT_PROJECT_ID))).toBe(false);
});
it("seeds the example beside Benchmarks the user already keeps, touching none of them", async () => {
const dir = benchmarksDir(tmpRoot, DEFAULT_PROJECT_ID);
// Creating a Benchmark makes benchmarks/ on its own, and an older build seeded only on
// initialization — so a Project can hold its own Benchmarks and no example. The example's
// own directory is the check, not the directory around it.
await fs.mkdir(path.join(dir, "swe-bench-v1"), { recursive: true });
await fs.writeFile(path.join(dir, "swe-bench-v1", "benchmark_config.toml"), 'title = "mine"\n');
await loadAgentState({ init: {} });
expect((await fs.readdir(dir)).sort()).toEqual([EXAMPLE_BENCHMARK_ID, "swe-bench-v1"]);
expect(await fs.readFile(path.join(dir, "swe-bench-v1", "benchmark_config.toml"), "utf8")).toBe(
'title = "mine"\n',
);
});
it("seeds the Project-level example even when the retired per-agent location holds one", async () => {
// A data root from before Benchmarks moved to the Project level keeps
// agents/default_agent/benchmarks/example-benchmark/; nothing reads it, and it must not
// stand in for the Project-level example.
const legacy = path.join(
agentStateDir(tmpRoot, DEFAULT_PROJECT_ID, DEFAULT_AGENT_ID),
"benchmarks",
EXAMPLE_BENCHMARK_ID,
);
await fs.mkdir(legacy, { recursive: true });
await fs.writeFile(path.join(legacy, "benchmark_config.toml"), 'title = "old"\n');
await loadAgentState({ init: {} });
const dir = benchmarksDir(tmpRoot, DEFAULT_PROJECT_ID);
expect(await exists(path.join(dir, EXAMPLE_BENCHMARK_ID, "benchmark_config.toml"))).toBe(true);
// The legacy copy is left exactly as it was.
expect(await fs.readFile(path.join(legacy, "benchmark_config.toml"), "utf8")).toBe(
'title = "old"\n',
);
});
it("loading default_agent re-creates the example when benchmarks/ is gone", async () => {
await loadAgentState({ init: {} });
const dir = benchmarksDir(tmpRoot, DEFAULT_PROJECT_ID);
await fs.rm(dir, { recursive: true, force: true });
// The load path, not the init path: an existing Agent is only read, and a data root that
// predates this provisioning gets its example on that first load.
await loadAgentState();
expect(await exists(path.join(dir, EXAMPLE_BENCHMARK_ID, "benchmark_config.toml"))).toBe(true);
});
it("loading default_agent writes the example back once it alone was deleted", async () => {
await loadAgentState({ init: {} });
const dir = benchmarksDir(tmpRoot, DEFAULT_PROJECT_ID);
await fs.rm(path.join(dir, EXAMPLE_BENCHMARK_ID), { recursive: true, force: true });
await loadAgentState();
// The example's own directory is the check, so a deletion lasts until the next load.
expect(await fs.readdir(dir)).toEqual([EXAMPLE_BENCHMARK_ID]);
});
});