Skip to content

A local-first multi-agent app development platform, built on PenguinHarness

12.5 KB
Raw Blame
/**
 * Provisioning of the example Benchmark.
 *
 * A Project without `benchmarks/example-benchmark/` gets it, at the Project level: two sample cases (each with statement/ and rubric/ indexed
 * by a README.md), benchmark_config.toml (runs = 2, status = published — the example is a
 * finished Benchmark, not one a Skill is still writing), and a scoreboard.yaml with three sample
 * evaluations, each labelled with default_agent as the Agent it tested — so the evaluation
 * center has data out of the box. Its description states plainly that this is a built-in
 * example and the whole directory can be deleted or replaced. Provisioning rides on
 * default_agent alone, on both its initialization and every later load; creating or loading an
 * ordinary Agent provisions nothing.
 *
 * The check is on the example's own directory, nothing else: whatever else `benchmarks/`
 * holds — the user's own Benchmarks, created before the example was ever seeded (creating one
 * makes the directory, and an older build seeded only on initialization) — and whatever an
 * older data root keeps at the retired per-agent location `agents/<agent>/benchmarks/`, which
 * this code never reads, the Project-level example is written when it is missing. That also
 * means deleting the example lasts until the next load of default_agent; an example that is
 * present is never touched, evaluations appended to it included.
 *
 * Scoring numbers follow the current Scoreboard contract: every Case is scored out of 100;
 * Case metrics are model-written Run averages and Evaluation metrics are model-written Case
 * averages. Cost ignores unknown values; Run cost preserves its recorded precision, Score
 * averages keep two decimals, cost averages keep six, and durations are rounded to integer
 * milliseconds.
 */
import fs from "node:fs/promises";
import path from "node:path";
import { stringify as stringifyToml } from "smol-toml";
import { stringify as stringifyYaml } from "yaml";
import { DEFAULT_AGENT_ID, benchmarksDir } from "./paths.js";

/** Directory name of the example Benchmark (the directory name is also its identifier). */
export const EXAMPLE_BENCHMARK_ID = "example-benchmark";

/** Contents of benchmark_config.toml (no model reference here — the model is recorded on each evaluation instead). */
const EXAMPLE_BENCHMARK_CONFIG = {
  title: "Example Benchmark",
  description:
    "A built-in example benchmark so the evaluation charts have data out of the box. " +
    "Replace it with your own.",
  runs: 2,
  status: "published",
};

/** Two sample cases: statement and scoring rubric (in English, 3-5 lines each). */
const EXAMPLE_CASES: Array<{ id: string; statement: string; rubric: string }> = [
  {
    id: "CASE-001-file-summary",
    statement: `# Task: Summarize a project file

Read the provided \`notes.txt\` in your workspace and write \`summary.md\` containing:
1. A one-paragraph overview of at most 3 sentences.
2. A bullet list of the three most important facts.
Keep the whole summary under 150 words.
`,
    rubric: `# Scoring rubric (max 100 points)

- 40 pts: \`summary.md\` exists and stays under 150 words.
- 40 pts: The three bullet facts are accurate and taken from \`notes.txt\`.
- 20 pts: The overview paragraph is coherent and at most 3 sentences.
Award partial credit per item; the case score is the sum.
`,
  },
  {
    id: "CASE-002-data-cleanup",
    statement: `# Task: Clean up a CSV dataset

The workspace contains \`users.csv\` with duplicate rows and inconsistent casing in the email column.
Produce \`users_clean.csv\` where:
1. Emails are lowercased and rows with an empty email are removed.
2. Exact duplicate rows are dropped, keeping the first occurrence.
Do not change the column order.
`,
    rubric: `# Scoring rubric (max 100 points)

- 40 pts: \`users_clean.csv\` exists and keeps the original column order.
- 40 pts: Emails are lowercased, empty-email rows removed, duplicates dropped (first kept).
- 20 pts: No unrelated rows or columns were modified.
Award partial credit per item; the case score is the sum.
`,
  },
];

/** Raw result of a single Run in the current Scoreboard format. */
interface ExampleRun {
  score: number;
  cost: number | null;
  duration_ms: number;
  session_id: string;
}

/**
 * Raw runs for the three sample evaluations (case-level and evaluation-level metrics are
 * computed from these, keeping the numbers self-consistent). Each carries the Agent it tested
 * and the model actually used for that round; the examples all evaluate default_agent on
 * deepseek-v4-pro at medium thinking, so they share one chart series.
 */
const EXAMPLE_EVALUATIONS: Array<{
  time: string;
  agent_id: string;
  version: number;
  provider: string;
  model_id: string;
  thinking_level: string;
  summary_title: string;
  summary: string;
  cases: Array<{ case: string; runs: ExampleRun[] }>;
}> = [
  {
    time: "2026-07-14T09:30:00Z",
    agent_id: DEFAULT_AGENT_ID,
    version: 1,
    provider: "deepseek",
    model_id: "deepseek-v4-pro",
    thinking_level: "medium",
    summary_title: "Baseline before any optimization",
    summary:
      "Example data (not a real evaluation): baseline scores of the built-in sample " +
      "benchmark before any optimization. Hypothesis for the next round: the agent skips " +
      "a final self-check, losing points on completeness.",
    cases: [
      {
        case: "CASE-001-file-summary",
        runs: [
          {
            score: 50,
            cost: 0.012,
            duration_ms: 42000,
            session_id: "session-2026-07-14-09-05-11-1a2b3c01",
          },
          {
            score: 70,
            cost: 0.014,
            duration_ms: 48000,
            session_id: "session-2026-07-14-09-13-27-1a2b3c02",
          },
        ],
      },
      {
        case: "CASE-002-data-cleanup",
        runs: [
          {
            score: 60,
            cost: null,
            duration_ms: 66000,
            session_id: "session-2026-07-14-09-21-45-1a2b3c03",
          },
          {
            score: 60,
            cost: null,
            duration_ms: 74000,
            session_id: "session-2026-07-14-09-28-52-1a2b3c04",
          },
        ],
      },
    ],
  },
  {
    time: "2026-07-15T09:30:00Z",
    agent_id: DEFAULT_AGENT_ID,
    version: 2,
    provider: "deepseek",
    model_id: "deepseek-v4-pro",
    thinking_level: "medium",
    summary_title: "Added an explicit planning step",
    summary:
      "Example data (not a real evaluation): after adding an explicit planning step to the " +
      "system prompt (hypothesis: written plans reduce missed requirements), both cases " +
      "improved. Next: tighten output formatting.",
    cases: [
      {
        case: "CASE-001-file-summary",
        runs: [
          {
            score: 70,
            cost: 0.011,
            duration_ms: 39000,
            session_id: "session-2026-07-15-09-04-33-2b3c4d01",
          },
          {
            score: 80,
            cost: 0.013,
            duration_ms: 45000,
            session_id: "session-2026-07-15-09-12-08-2b3c4d02",
          },
        ],
      },
      {
        case: "CASE-002-data-cleanup",
        runs: [
          {
            score: 70,
            cost: 0.016,
            duration_ms: 60000,
            session_id: "session-2026-07-15-09-19-40-2b3c4d03",
          },
          {
            score: 80,
            cost: 0.02,
            duration_ms: 68000,
            session_id: "session-2026-07-15-09-26-59-2b3c4d04",
          },
        ],
      },
    ],
  },
  {
    time: "2026-07-16T09:30:00Z",
    agent_id: DEFAULT_AGENT_ID,
    version: 3,
    provider: "deepseek",
    model_id: "deepseek-v4-pro",
    thinking_level: "medium",
    summary_title: "Verify deliverables before finishing",
    summary:
      "Example data (not a real evaluation): after instructing the agent to verify its " +
      "deliverables against the statement before finishing (hypothesis: a final check " +
      "catches formatting slips), scores improved again. Replace this benchmark with your " +
      "own to track real progress.",
    cases: [
      {
        case: "CASE-001-file-summary",
        runs: [
          {
            score: 80,
            cost: 0.01,
            duration_ms: 36000,
            session_id: "session-2026-07-16-09-03-21-3c4d5e01",
          },
          {
            score: 90,
            cost: 0.012,
            duration_ms: 40000,
            session_id: "session-2026-07-16-09-10-46-3c4d5e02",
          },
        ],
      },
      {
        case: "CASE-002-data-cleanup",
        runs: [
          {
            score: 90,
            cost: 0.015,
            duration_ms: 55000,
            session_id: "session-2026-07-16-09-18-02-3c4d5e03",
          },
          {
            score: 80,
            cost: 0.017,
            duration_ms: 61000,
            session_id: "session-2026-07-16-09-25-30-3c4d5e04",
          },
        ],
      },
    ],
  },
];

function roundTwo(v: number): number {
  return Math.round(v * 100) / 100;
}

function averageTwo(values: number[]): number {
  return roundTwo(values.reduce((a, b) => a + b, 0) / values.length);
}

function averageSix(values: number[]): number {
  return Math.round((values.reduce((a, b) => a + b, 0) / values.length) * 1_000_000) / 1_000_000;
}

function averageDuration(values: number[]): number {
  return Math.round(values.reduce((a, b) => a + b, 0) / values.length);
}

function averageKnownCost(values: Array<number | null>): number | null {
  const known = values.filter((value): value is number => value !== null);
  return known.length > 0 ? averageSix(known) : null;
}

/**
 * Builds the example Scoreboard exactly as the model is instructed to write it. This helper
 * exists only to provision deterministic sample data; runtime readers trust the stored values
 * and never call it to recompute a user Scoreboard.
 */
export function buildExampleScoreboard(): {
  evaluations: Array<{
    time: string;
    agent_id: string;
    version: number;
    provider: string;
    model_id: string;
    thinking_level: string;
    summary_title: string;
    summary: string;
    score: number;
    cost: number | null;
    duration_ms: number;
    cases: Array<{
      case: string;
      score: number;
      cost: number | null;
      duration_ms: number;
      runs: ExampleRun[];
    }>;
  }>;
} {
  return {
    evaluations: EXAMPLE_EVALUATIONS.map((e) => {
      const cases = e.cases.map((c) => ({
        case: c.case,
        score: averageTwo(c.runs.map((r) => r.score)),
        cost: averageKnownCost(c.runs.map((r) => r.cost)),
        duration_ms: averageDuration(c.runs.map((r) => r.duration_ms)),
        runs: c.runs,
      }));
      return {
        time: e.time,
        agent_id: e.agent_id,
        version: e.version,
        provider: e.provider,
        model_id: e.model_id,
        thinking_level: e.thinking_level,
        summary_title: e.summary_title,
        summary: e.summary,
        score: averageTwo(cases.map((c) => c.score)),
        cost: averageKnownCost(cases.map((c) => c.cost)),
        duration_ms: averageDuration(cases.map((c) => c.duration_ms)),
        cases,
      };
    }),
  };
}

/**
 * Provisions the example Benchmark into the Project's `benchmarks/`: when
 * `benchmarks/example-benchmark/` does not exist, creates it (config, the two sample cases, and
 * the scoreboard), making `benchmarks/` on the way if needed; when it does, does nothing. The
 * user's own Benchmarks beside it, and anything at the retired per-agent location, play no
 * part. Callers are restricted to default_agent's initialization and load paths (see
 * agent-state.ts).
 */
export async function provisionExampleBenchmark(root: string, projectId: string): Promise<void> {
  const benchDir = path.join(benchmarksDir(root, projectId), EXAMPLE_BENCHMARK_ID);
  try {
    await fs.access(benchDir);
    return;
  } catch {
    // The Project has no example: proceed with provisioning.
  }
  await Promise.all(
    EXAMPLE_CASES.flatMap((c) => [
      fs.mkdir(path.join(benchDir, c.id, "statement"), { recursive: true }),
      fs.mkdir(path.join(benchDir, c.id, "rubric"), { recursive: true }),
    ]),
  );
  await Promise.all([
    fs.writeFile(
      path.join(benchDir, "benchmark_config.toml"),
      `${stringifyToml(EXAMPLE_BENCHMARK_CONFIG)}\n`,
      "utf8",
    ),
    fs.writeFile(
      path.join(benchDir, "scoreboard.yaml"),
      stringifyYaml(buildExampleScoreboard()),
      "utf8",
    ),
    ...EXAMPLE_CASES.flatMap((c) => [
      fs.writeFile(path.join(benchDir, c.id, "statement", "README.md"), c.statement, "utf8"),
      fs.writeFile(path.join(benchDir, c.id, "rubric", "README.md"), c.rubric, "utf8"),
    ]),
  ]);
}