Skip to content

A local-first multi-agent app development platform, built on PenguinHarness

19.0 KB
Raw Blame
/**
 * Benchmark score reading: walks the Project's `benchmarks/<id>/`, reads
 * `benchmark_config.toml` (title, description, per-case run count `runs`, and the build
 * `status`: `draft` while the Benchmark is still being written, `failed` when its calibration
 * never produced a result to freeze, `published` otherwise) and `scoreboard.yaml`
 * (evaluations[], each carrying the Agent it tested, each case its model-written averages and
 * a runs array).
 * Content is normally created and refined by the benchmark-design Skill; the server also
 * writes the same layout for a Benchmark created by hand (`create`) and removes a Benchmark
 * directory whole (`remove`), and never touches a scoreboard.
 * `benchmark_config.toml` is what makes a directory a Benchmark: `list` skips one without it.
 * Files that are there but corrupt degrade gracefully (title falls back to the directory
 * name, scores come back empty) rather than throwing.
 *
 * Case and Evaluation averages are authoritative file values. The server validates
 * the current shape but never recomputes aggregates and does not migrate or backfill
 * old Scoreboard formats.
 * Docs: /docs/self-improvement § "Benchmark storage".
 */
import fs from "node:fs/promises";
import path from "node:path";
import { parse as parseToml, stringify as stringifyToml } from "smol-toml";
import { parse as parseYaml, stringify as stringifyYaml } from "yaml";
import { benchmarksDir } from "@adelie/core";
import type {
  BenchmarkCaseScore,
  BenchmarkCaseSummary,
  BenchmarkCasesResponse,
  BenchmarkEvaluation,
  BenchmarkRunScore,
  BenchmarkStatus,
  BenchmarkSummary,
  BenchmarksResponse,
  CaseMaterial,
  WorkspaceFilesResponse,
} from "../api/types.js";
import type { WorkspaceFileContent, WorkspaceFileReadOptions } from "./workspace-files-service.js";
import { HttpError } from "../http/errors.js";
import { Component, Use } from "@adelie/core/kernel";
import type { Paths } from "../hmr/capabilities.js";
import type { Benchmarks } from "../mechanisms/agents.js";
import type { WorkspaceFiles } from "../mechanisms/workspace.js";

const STATEMENT_TITLE_READ_BYTES = 64 * 1024;

/** One case of a hand-made Benchmark; ids are validated by the route before they reach the filesystem. */
export interface BenchmarkCaseInput {
  id: string;
  title: string;
  statement: string;
  rubric: string;
}

export interface BenchmarkCreateInput {
  id: string;
  title: string;
  description?: string;
  runs: number;
  cases: BenchmarkCaseInput[];
}

function asRecord(v: unknown): Record<string, unknown> {
  return v !== null && typeof v === "object" && !Array.isArray(v)
    ? (v as Record<string, unknown>)
    : {};
}

function numberOr(v: unknown): number | undefined {
  return typeof v === "number" && Number.isFinite(v) ? v : undefined;
}

function scoreOr(v: unknown): number | undefined {
  const value = numberOr(v);
  return value !== undefined && value >= 0 && value <= 100 ? value : undefined;
}

function nonNegativeOr(v: unknown): number | undefined {
  const value = numberOr(v);
  return value !== undefined && value >= 0 ? value : undefined;
}

function nonNegativeIntegerOr(v: unknown): number | undefined {
  const value = nonNegativeOr(v);
  return value !== undefined && Number.isInteger(value) ? value : undefined;
}

/** `null` is the one valid unknown-cost representation; undefined means invalid input. */
function nullableCostOr(v: unknown): number | null | undefined {
  if (v === null) return null;
  return nonNegativeOr(v);
}

function stringOr(v: unknown): string | undefined {
  return typeof v === "string" && v !== "" ? v : undefined;
}

/**
 * The Agent under test on one evaluation. Unlike the runtime fields it never invalidates a
 * record: a scoreboard written before evaluations carried one still displays, unlabelled.
 */
function agentIdOr(v: unknown): string | null {
  const value = typeof v === "string" ? v.trim() : "";
  return value !== "" ? value : null;
}

function isWithin(parent: string, child: string): boolean {
  const relative = path.relative(parent, child);
  return relative !== "" && !relative.startsWith("..") && !path.isAbsolute(relative);
}

function statementTitle(statement: string, fallback: string): string {
  const heading = /^#\s+(.+)$/m.exec(statement)?.[1]?.trim();
  return heading?.replace(/^Case\s+\d+\s*:\s*/i, "") || fallback;
}

async function readStatementTitle(readme: string, fallback: string): Promise<string> {
  const handle = await fs.open(readme, "r");
  try {
    const buffer = Buffer.alloc(STATEMENT_TITLE_READ_BYTES);
    const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0);
    return statementTitle(buffer.subarray(0, bytesRead).toString("utf8"), fallback);
  } finally {
    await handle.close();
  }
}

/** Shapes one current-format Run; a malformed entry invalidates its containing Case. */
function toRun(v: unknown): BenchmarkRunScore | null {
  const r = asRecord(v);
  const score = scoreOr(r.score);
  const cost = nullableCostOr(r.cost);
  const durationMs = nonNegativeIntegerOr(r.duration_ms);
  const sessionId = stringOr(r.session_id);
  if (score === undefined || cost === undefined || durationMs === undefined || !sessionId)
    return null;
  return { score, cost, durationMs, sessionId };
}

/**
 * Shapes one current-format Case. Its stored aggregates are trusted as written:
 * this parser intentionally performs no average or consistency calculation.
 */
function toCase(v: unknown): BenchmarkCaseScore | null {
  const cr = asRecord(v);
  const caseId = stringOr(cr.case);
  const score = scoreOr(cr.score);
  const cost = nullableCostOr(cr.cost);
  const durationMs = nonNegativeIntegerOr(cr.duration_ms);
  if (
    !caseId ||
    score === undefined ||
    cost === undefined ||
    durationMs === undefined ||
    "max_score" in cr ||
    !Array.isArray(cr.runs) ||
    cr.runs.length === 0
  ) {
    return null;
  }
  const parsedRuns = cr.runs.map(toRun);
  if (parsedRuns.some((run) => run === null)) return null;
  const runs = parsedRuns as BenchmarkRunScore[];
  return {
    case: caseId,
    score,
    cost,
    durationMs,
    runs,
  };
}

/** Shapes one current-format Evaluation and trusts its stored aggregate metrics. */
function toEvaluation(v: unknown): BenchmarkEvaluation | null {
  const r = asRecord(v);
  const time = r.time instanceof Date ? r.time.toISOString() : r.time;
  const agentId = agentIdOr(r.agent_id);
  const score = scoreOr(r.score);
  const cost = nullableCostOr(r.cost);
  const durationMs = nonNegativeIntegerOr(r.duration_ms);
  const summary = stringOr(r.summary);
  // Title and body are separate: summary_title is a one-line
  // conclusion, summary is the body text.
  const summaryTitle = stringOr(r.summary_title);
  const modelId = stringOr(r.model_id);
  const provider = stringOr(r.provider);
  const thinkingLevel = stringOr(r.thinking_level);
  const version = nonNegativeIntegerOr(r.version);
  if (
    typeof time !== "string" ||
    time === "" ||
    score === undefined ||
    cost === undefined ||
    durationMs === undefined ||
    !modelId ||
    !provider ||
    !thinkingLevel ||
    version === undefined ||
    version < 1 ||
    !Array.isArray(r.cases) ||
    r.cases.length === 0
  ) {
    return null;
  }
  const parsedCases = r.cases.map(toCase);
  if (parsedCases.some((item) => item === null)) return null;
  const cases = parsedCases as BenchmarkCaseScore[];
  return {
    time,
    agentId,
    ...(summaryTitle !== undefined ? { summaryTitle } : {}),
    ...(summary !== undefined ? { summary } : {}),
    modelId,
    provider,
    thinkingLevel,
    score,
    version,
    cost,
    durationMs,
    cases,
  };
}

@Component()
export class BenchmarkService implements Benchmarks {
  @Use() private readonly paths!: Paths;
  private get root(): string {
    return this.paths.root;
  }
  @Use() private readonly workspaceFiles!: WorkspaceFiles;

  async list(projectId: string): Promise<BenchmarksResponse> {
    const dir = benchmarksDir(this.root, projectId);
    let items: Array<{ name: string; isDir: boolean }>;
    try {
      const entries = await fs.readdir(dir, { withFileTypes: true });
      items = entries.map((e) => ({ name: e.name, isDir: e.isDirectory() }));
    } catch {
      return { benchmarks: [] }; // Doesn't exist when unconfigured.
    }
    const benchmarks: BenchmarkSummary[] = [];
    for (const item of items.filter((i) => i.isDir).sort((a, b) => a.name.localeCompare(b.name))) {
      const benchDir = path.join(dir, item.name);
      // Only `benchmark_config.toml` makes a directory a Benchmark — it is the file the
      // evaluation Skills require, and without it there is no title and no run count. A
      // Benchmark deleted while an evaluation is still running comes back as the paths that
      // run keeps writing, config not among them; that debris is not a Benchmark and is not
      // listed. Absence of results is not absence of a Benchmark: one that has never run has
      // its config and lists as usual.
      try {
        await fs.access(path.join(benchDir, "benchmark_config.toml"));
      } catch (error) {
        if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
        continue;
      }
      benchmarks.push(await this.readBenchmark(benchDir, item.name));
    }
    return { benchmarks };
  }

  /**
   * Every id `create` refuses as taken, as names only: each entry under `benchmarks/`. Unlike
   * `list`, this includes a directory with no `benchmark_config.toml` (the debris of one deleted
   * mid-evaluation), and it reads no config and no scoreboard.
   */
  async takenIds(projectId: string): Promise<string[]> {
    try {
      return await fs.readdir(benchmarksDir(this.root, projectId));
    } catch (error) {
      if ((error as NodeJS.ErrnoException).code === "ENOENT") return [];
      throw error;
    }
  }

  /**
   * Creates `benchmarks/<id>/` in the layout the evaluation Skills read: `benchmark_config.toml`
   * (title, description, runs, status), `scoreboard.yaml` with an empty evaluations list, and
   * per case `statement/README.md` (`# <title>`, then the statement) and `rubric/README.md` (the
   * rubric verbatim). An existing directory is a 409, never merged into: a Benchmark's scores stay
   * comparable only while its cases are rewritten by nothing but the Skills. A half-written
   * directory is removed again when a later write fails.
   */
  async create(projectId: string, input: BenchmarkCreateInput): Promise<BenchmarkSummary> {
    const dir = benchmarksDir(this.root, projectId);
    const benchDir = path.join(dir, input.id);
    await fs.mkdir(dir, { recursive: true });
    try {
      // A non-recursive mkdir is the existence check: it fails atomically on a directory
      // that is already there, so two creates of one id cannot both proceed.
      await fs.mkdir(benchDir);
    } catch (error) {
      if ((error as NodeJS.ErrnoException).code === "EEXIST") {
        throw new HttpError(409, "benchmark_exists", `Benchmark already exists: ${input.id}`);
      }
      throw error;
    }
    try {
      const config = {
        title: input.title,
        ...(input.description !== undefined && input.description !== ""
          ? { description: input.description }
          : {}),
        runs: input.runs,
        // A Benchmark made by hand is complete the moment it is submitted: its cases are
        // written and frozen, so nothing is left for a Skill to finish.
        status: "published",
      };
      await fs.writeFile(
        path.join(benchDir, "benchmark_config.toml"),
        `${stringifyToml(config)}\n`,
        "utf8",
      );
      await fs.writeFile(
        path.join(benchDir, "scoreboard.yaml"),
        stringifyYaml({ evaluations: [] }),
        "utf8",
      );
      for (const item of input.cases) {
        const caseDir = path.join(benchDir, item.id);
        await fs.mkdir(path.join(caseDir, "statement"), { recursive: true });
        await fs.mkdir(path.join(caseDir, "rubric"), { recursive: true });
        await fs.writeFile(
          path.join(caseDir, "statement", "README.md"),
          `# ${item.title.trim()}\n\n${item.statement.trim()}\n`,
          "utf8",
        );
        await fs.writeFile(
          path.join(caseDir, "rubric", "README.md"),
          `${item.rubric.trim()}\n`,
          "utf8",
        );
      }
    } catch (error) {
      await fs.rm(benchDir, { recursive: true, force: true });
      throw error;
    }
    return this.readBenchmark(benchDir, input.id);
  }

  /**
   * Removes `benchmarks/<id>/` whole — cases, config and scoreboard. Only a real directory
   * counts as existing: a symlink there is not followed, so nothing outside the Project's own
   * benchmarks directory can be deleted through this route.
   */
  async remove(projectId: string, benchmarkId: string): Promise<void> {
    const benchDir = path.join(benchmarksDir(this.root, projectId), benchmarkId);
    let isDirectory = false;
    try {
      isDirectory = (await fs.lstat(benchDir)).isDirectory();
    } catch {
      // Missing: reported below as not found.
    }
    if (!isDirectory) {
      throw new HttpError(404, "not_found", `Benchmark does not exist: ${benchmarkId}`);
    }
    await fs.rm(benchDir, { recursive: true, force: true });
  }

  async listCases(projectId: string, benchmarkId: string): Promise<BenchmarkCasesResponse> {
    const baseDir = benchmarksDir(this.root, projectId);
    const benchDir = path.join(baseDir, benchmarkId);
    let entries: Array<{ name: string; isDirectory(): boolean }>;
    let realBaseDir: string;
    let realBenchDir: string;
    try {
      [entries, realBaseDir, realBenchDir] = await Promise.all([
        fs.readdir(benchDir, { withFileTypes: true }),
        fs.realpath(baseDir),
        fs.realpath(benchDir),
      ]);
    } catch {
      return { cases: [] };
    }
    if (!isWithin(realBaseDir, realBenchDir)) return { cases: [] };

    const cases: BenchmarkCaseSummary[] = [];
    for (const entry of entries
      .filter((item) => item.isDirectory() && item.name.startsWith("CASE-"))
      .sort((a, b) => a.name.localeCompare(b.name))) {
      const fallback: BenchmarkCaseSummary = { id: entry.name, title: entry.name };
      try {
        const statementDir = await this.caseMaterialRoot(
          projectId,
          benchmarkId,
          entry.name,
          "statement",
        );
        const realReadme = await fs.realpath(path.join(statementDir, "README.md"));
        if (!isWithin(statementDir, realReadme)) throw new Error("README escapes Statement");
        cases.push({
          id: entry.name,
          title: await readStatementTitle(realReadme, entry.name),
        });
      } catch {
        cases.push(fallback);
      }
    }
    return { cases };
  }

  async listCaseFiles(
    projectId: string,
    benchmarkId: string,
    caseId: string,
    rel: string,
    material: CaseMaterial,
  ): Promise<WorkspaceFilesResponse> {
    const materialRoot = await this.caseMaterialRoot(projectId, benchmarkId, caseId, material);
    return this.workspaceFiles.list(materialRoot, rel);
  }

  async readCaseFile(
    projectId: string,
    benchmarkId: string,
    caseId: string,
    rel: string,
    material: CaseMaterial,
    options?: WorkspaceFileReadOptions,
  ): Promise<WorkspaceFileContent> {
    const materialRoot = await this.caseMaterialRoot(projectId, benchmarkId, caseId, material);
    return this.workspaceFiles.read(materialRoot, rel, options);
  }

  private async caseMaterialRoot(
    projectId: string,
    benchmarkId: string,
    caseId: string,
    material: CaseMaterial,
  ): Promise<string> {
    const benchDir = path.join(benchmarksDir(this.root, projectId), benchmarkId);
    const caseDir = path.join(benchDir, caseId);
    const materialRoot = path.join(caseDir, material);
    try {
      const [realBenchDir, realCaseDir, realMaterialRoot] = await Promise.all([
        fs.realpath(benchDir),
        fs.realpath(caseDir),
        fs.realpath(materialRoot),
      ]);
      if (
        !isWithin(realBenchDir, realCaseDir) ||
        path.dirname(realMaterialRoot) !== realCaseDir ||
        path.basename(realMaterialRoot) !== material
      ) {
        throw new Error("Case material is not canonical");
      }
      return realMaterialRoot;
    } catch {
      throw new HttpError(404, "not_found", `Case ${material} does not exist.`);
    }
  }

  private async readBenchmark(benchDir: string, id: string): Promise<BenchmarkSummary> {
    // benchmark_config.toml: title, description, per-case run count and build status (falls
    // back to defaults if corrupt). The model isn't part of the config — each evaluation
    // carries the Model actually used for that run.
    let title = id;
    let description: string | undefined;
    let runs: number | undefined;
    let status: BenchmarkStatus = "published";
    try {
      const config = asRecord(
        parseToml(await fs.readFile(path.join(benchDir, "benchmark_config.toml"), "utf8")),
      );
      if (typeof config.title === "string" && config.title !== "") title = config.title;
      if (typeof config.description === "string" && config.description !== "") {
        description = config.description;
      }
      const configRuns = numberOr(config.runs);
      if (configRuns !== undefined && Number.isInteger(configRuns) && configRuns >= 1) {
        runs = configRuns;
      }
      // The two states that make a Benchmark unusable are literal: "draft" while it is still
      // being built, "failed" when its calibration never produced a result to freeze. A config
      // written before this field existed has none, and an unrecognized value is neither, so
      // both read as published.
      const raw = stringOr(config.status);
      status = raw === "draft" ? "draft" : raw === "failed" ? "failed" : "published";
    } catch {
      // Missing or corrupt: title falls back to the directory name.
    }

    // scoreboard.yaml: evaluations[] is appended over time; bad entries are dropped one by one.
    let evaluations: BenchmarkEvaluation[] = [];
    try {
      const scoreboard = asRecord(
        parseYaml(await fs.readFile(path.join(benchDir, "scoreboard.yaml"), "utf8")),
      );
      if (Array.isArray(scoreboard.evaluations)) {
        evaluations = scoreboard.evaluations
          .map(toEvaluation)
          .filter((e): e is BenchmarkEvaluation => e !== null);
      }
    } catch {
      // No scores yet.
    }

    // Case count: number of semantic Case subfolders.
    let caseCount = 0;
    try {
      const entries = await fs.readdir(benchDir, { withFileTypes: true });
      caseCount = entries.filter((e) => e.isDirectory() && e.name.startsWith("CASE-")).length;
    } catch {
      // Stays at 0.
    }

    return {
      id,
      title,
      ...(description !== undefined ? { description } : {}),
      ...(runs !== undefined ? { runs } : {}),
      status,
      caseCount,
      evaluations,
      // Which Agents this Benchmark has evaluated is a fact of its scoreboard, not of its
      // config: first-seen order, so the list reads in the order the Agents were tested.
      agentIds: [
        ...new Set(
          evaluations
            .map((evaluation) => evaluation.agentId)
            .filter((agentId): agentId is string => agentId !== null),
        ),
      ],
    };
  }
}