A local-first multi-agent app development platform, built on PenguinHarness
/**
* Provisioning of the example Benchmark.
*
* A Project without `benchmarks/example-benchmark/` gets it, at the Project level: two sample cases (each with statement/ and rubric/ indexed
* by a README.md), benchmark_config.toml (runs = 2, status = published — the example is a
* finished Benchmark, not one a Skill is still writing), and a scoreboard.yaml with three sample
* evaluations, each labelled with default_agent as the Agent it tested — so the evaluation
* center has data out of the box. Its description states plainly that this is a built-in
* example and the whole directory can be deleted or replaced. Provisioning rides on
* default_agent alone, on both its initialization and every later load; creating or loading an
* ordinary Agent provisions nothing.
*
* The check is on the example's own directory, nothing else: whatever else `benchmarks/`
* holds — the user's own Benchmarks, created before the example was ever seeded (creating one
* makes the directory, and an older build seeded only on initialization) — and whatever an
* older data root keeps at the retired per-agent location `agents/<agent>/benchmarks/`, which
* this code never reads, the Project-level example is written when it is missing. That also
* means deleting the example lasts until the next load of default_agent; an example that is
* present is never touched, evaluations appended to it included.
*
* Scoring numbers follow the current Scoreboard contract: every Case is scored out of 100;
* Case metrics are model-written Run averages and Evaluation metrics are model-written Case
* averages. Cost ignores unknown values; Run cost preserves its recorded precision, Score
* averages keep two decimals, cost averages keep six, and durations are rounded to integer
* milliseconds.
*/
import fs from "node:fs/promises";
import path from "node:path";
import { stringify as stringifyToml } from "smol-toml";
import { stringify as stringifyYaml } from "yaml";
import { DEFAULT_AGENT_ID, benchmarksDir } from "./paths.js";
/** Directory name of the example Benchmark (the directory name is also its identifier). */
export const EXAMPLE_BENCHMARK_ID = "example-benchmark";
/** Contents of benchmark_config.toml (no model reference here — the model is recorded on each evaluation instead). */
const EXAMPLE_BENCHMARK_CONFIG = {
title: "Example Benchmark",
description:
"A built-in example benchmark so the evaluation charts have data out of the box. " +
"Replace it with your own.",
runs: 2,
status: "published",
};
/** Two sample cases: statement and scoring rubric (in English, 3-5 lines each). */
const EXAMPLE_CASES: Array<{ id: string; statement: string; rubric: string }> = [
{
id: "CASE-001-file-summary",
statement: `# Task: Summarize a project file
Read the provided \`notes.txt\` in your workspace and write \`summary.md\` containing:
1. A one-paragraph overview of at most 3 sentences.
2. A bullet list of the three most important facts.
Keep the whole summary under 150 words.
`,
rubric: `# Scoring rubric (max 100 points)
- 40 pts: \`summary.md\` exists and stays under 150 words.
- 40 pts: The three bullet facts are accurate and taken from \`notes.txt\`.
- 20 pts: The overview paragraph is coherent and at most 3 sentences.
Award partial credit per item; the case score is the sum.
`,
},
{
id: "CASE-002-data-cleanup",
statement: `# Task: Clean up a CSV dataset
The workspace contains \`users.csv\` with duplicate rows and inconsistent casing in the email column.
Produce \`users_clean.csv\` where:
1. Emails are lowercased and rows with an empty email are removed.
2. Exact duplicate rows are dropped, keeping the first occurrence.
Do not change the column order.
`,
rubric: `# Scoring rubric (max 100 points)
- 40 pts: \`users_clean.csv\` exists and keeps the original column order.
- 40 pts: Emails are lowercased, empty-email rows removed, duplicates dropped (first kept).
- 20 pts: No unrelated rows or columns were modified.
Award partial credit per item; the case score is the sum.
`,
},
];
/** Raw result of a single Run in the current Scoreboard format. */
interface ExampleRun {
score: number;
cost: number | null;
duration_ms: number;
session_id: string;
}
/**
* Raw runs for the three sample evaluations (case-level and evaluation-level metrics are
* computed from these, keeping the numbers self-consistent). Each carries the Agent it tested
* and the model actually used for that round; the examples all evaluate default_agent on
* deepseek-v4-pro at medium thinking, so they share one chart series.
*/
const EXAMPLE_EVALUATIONS: Array<{
time: string;
agent_id: string;
version: number;
provider: string;
model_id: string;
thinking_level: string;
summary_title: string;
summary: string;
cases: Array<{ case: string; runs: ExampleRun[] }>;
}> = [
{
time: "2026-07-14T09:30:00Z",
agent_id: DEFAULT_AGENT_ID,
version: 1,
provider: "deepseek",
model_id: "deepseek-v4-pro",
thinking_level: "medium",
summary_title: "Baseline before any optimization",
summary:
"Example data (not a real evaluation): baseline scores of the built-in sample " +
"benchmark before any optimization. Hypothesis for the next round: the agent skips " +
"a final self-check, losing points on completeness.",
cases: [
{
case: "CASE-001-file-summary",
runs: [
{
score: 50,
cost: 0.012,
duration_ms: 42000,
session_id: "session-2026-07-14-09-05-11-1a2b3c01",
},
{
score: 70,
cost: 0.014,
duration_ms: 48000,
session_id: "session-2026-07-14-09-13-27-1a2b3c02",
},
],
},
{
case: "CASE-002-data-cleanup",
runs: [
{
score: 60,
cost: null,
duration_ms: 66000,
session_id: "session-2026-07-14-09-21-45-1a2b3c03",
},
{
score: 60,
cost: null,
duration_ms: 74000,
session_id: "session-2026-07-14-09-28-52-1a2b3c04",
},
],
},
],
},
{
time: "2026-07-15T09:30:00Z",
agent_id: DEFAULT_AGENT_ID,
version: 2,
provider: "deepseek",
model_id: "deepseek-v4-pro",
thinking_level: "medium",
summary_title: "Added an explicit planning step",
summary:
"Example data (not a real evaluation): after adding an explicit planning step to the " +
"system prompt (hypothesis: written plans reduce missed requirements), both cases " +
"improved. Next: tighten output formatting.",
cases: [
{
case: "CASE-001-file-summary",
runs: [
{
score: 70,
cost: 0.011,
duration_ms: 39000,
session_id: "session-2026-07-15-09-04-33-2b3c4d01",
},
{
score: 80,
cost: 0.013,
duration_ms: 45000,
session_id: "session-2026-07-15-09-12-08-2b3c4d02",
},
],
},
{
case: "CASE-002-data-cleanup",
runs: [
{
score: 70,
cost: 0.016,
duration_ms: 60000,
session_id: "session-2026-07-15-09-19-40-2b3c4d03",
},
{
score: 80,
cost: 0.02,
duration_ms: 68000,
session_id: "session-2026-07-15-09-26-59-2b3c4d04",
},
],
},
],
},
{
time: "2026-07-16T09:30:00Z",
agent_id: DEFAULT_AGENT_ID,
version: 3,
provider: "deepseek",
model_id: "deepseek-v4-pro",
thinking_level: "medium",
summary_title: "Verify deliverables before finishing",
summary:
"Example data (not a real evaluation): after instructing the agent to verify its " +
"deliverables against the statement before finishing (hypothesis: a final check " +
"catches formatting slips), scores improved again. Replace this benchmark with your " +
"own to track real progress.",
cases: [
{
case: "CASE-001-file-summary",
runs: [
{
score: 80,
cost: 0.01,
duration_ms: 36000,
session_id: "session-2026-07-16-09-03-21-3c4d5e01",
},
{
score: 90,
cost: 0.012,
duration_ms: 40000,
session_id: "session-2026-07-16-09-10-46-3c4d5e02",
},
],
},
{
case: "CASE-002-data-cleanup",
runs: [
{
score: 90,
cost: 0.015,
duration_ms: 55000,
session_id: "session-2026-07-16-09-18-02-3c4d5e03",
},
{
score: 80,
cost: 0.017,
duration_ms: 61000,
session_id: "session-2026-07-16-09-25-30-3c4d5e04",
},
],
},
],
},
];
function roundTwo(v: number): number {
return Math.round(v * 100) / 100;
}
function averageTwo(values: number[]): number {
return roundTwo(values.reduce((a, b) => a + b, 0) / values.length);
}
function averageSix(values: number[]): number {
return Math.round((values.reduce((a, b) => a + b, 0) / values.length) * 1_000_000) / 1_000_000;
}
function averageDuration(values: number[]): number {
return Math.round(values.reduce((a, b) => a + b, 0) / values.length);
}
function averageKnownCost(values: Array<number | null>): number | null {
const known = values.filter((value): value is number => value !== null);
return known.length > 0 ? averageSix(known) : null;
}
/**
* Builds the example Scoreboard exactly as the model is instructed to write it. This helper
* exists only to provision deterministic sample data; runtime readers trust the stored values
* and never call it to recompute a user Scoreboard.
*/
export function buildExampleScoreboard(): {
evaluations: Array<{
time: string;
agent_id: string;
version: number;
provider: string;
model_id: string;
thinking_level: string;
summary_title: string;
summary: string;
score: number;
cost: number | null;
duration_ms: number;
cases: Array<{
case: string;
score: number;
cost: number | null;
duration_ms: number;
runs: ExampleRun[];
}>;
}>;
} {
return {
evaluations: EXAMPLE_EVALUATIONS.map((e) => {
const cases = e.cases.map((c) => ({
case: c.case,
score: averageTwo(c.runs.map((r) => r.score)),
cost: averageKnownCost(c.runs.map((r) => r.cost)),
duration_ms: averageDuration(c.runs.map((r) => r.duration_ms)),
runs: c.runs,
}));
return {
time: e.time,
agent_id: e.agent_id,
version: e.version,
provider: e.provider,
model_id: e.model_id,
thinking_level: e.thinking_level,
summary_title: e.summary_title,
summary: e.summary,
score: averageTwo(cases.map((c) => c.score)),
cost: averageKnownCost(cases.map((c) => c.cost)),
duration_ms: averageDuration(cases.map((c) => c.duration_ms)),
cases,
};
}),
};
}
/**
* Provisions the example Benchmark into the Project's `benchmarks/`: when
* `benchmarks/example-benchmark/` does not exist, creates it (config, the two sample cases, and
* the scoreboard), making `benchmarks/` on the way if needed; when it does, does nothing. The
* user's own Benchmarks beside it, and anything at the retired per-agent location, play no
* part. Callers are restricted to default_agent's initialization and load paths (see
* agent-state.ts).
*/
export async function provisionExampleBenchmark(root: string, projectId: string): Promise<void> {
const benchDir = path.join(benchmarksDir(root, projectId), EXAMPLE_BENCHMARK_ID);
try {
await fs.access(benchDir);
return;
} catch {
// The Project has no example: proceed with provisioning.
}
await Promise.all(
EXAMPLE_CASES.flatMap((c) => [
fs.mkdir(path.join(benchDir, c.id, "statement"), { recursive: true }),
fs.mkdir(path.join(benchDir, c.id, "rubric"), { recursive: true }),
]),
);
await Promise.all([
fs.writeFile(
path.join(benchDir, "benchmark_config.toml"),
`${stringifyToml(EXAMPLE_BENCHMARK_CONFIG)}\n`,
"utf8",
),
fs.writeFile(
path.join(benchDir, "scoreboard.yaml"),
stringifyYaml(buildExampleScoreboard()),
"utf8",
),
...EXAMPLE_CASES.flatMap((c) => [
fs.writeFile(path.join(benchDir, c.id, "statement", "README.md"), c.statement, "utf8"),
fs.writeFile(path.join(benchDir, c.id, "rubric", "README.md"), c.rubric, "utf8"),
]),
]);
}