Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

2.6 KB
Raw Blame
#!/usr/bin/env node

import { existsSync, readFileSync } from "node:fs";
import { dirname, join, resolve } from "node:path";
import { parseArgs } from "node:util";

const { positionals, values } = parseArgs({
  args: process.argv.slice(2),
  allowPositionals: true,
  options: { json: { type: "boolean" }, help: { type: "boolean", short: "h" } },
  strict: true,
});

if (values.help || positionals.length !== 1) {
  console.log("Usage: node scripts/timings.mjs <run-directory-or-timings.json> [--json]");
  process.exit(values.help ? 0 : 1);
}

const timingsPath = resolveTimingsPath(positionals[0]);
const timings = JSON.parse(readFileSync(timingsPath, "utf8"));
if (!Array.isArray(timings)) throw new Error(`${timingsPath} must contain a timing array.`);

if (values.json) {
  console.log(JSON.stringify(timings, null, 2));
  process.exit(0);
}

const phases = timings.filter((timing) => timing.phase !== "run.context");
const width = Math.max(...phases.map((timing) => timing.phase.length));
for (const timing of phases) {
  const details = timing.details === undefined ? "" : ` ${JSON.stringify(timing.details)}`;
  console.log(
    `${timing.phase.padEnd(width)}  ${formatDuration(timing.durationMs).padStart(8)}  ${timing.outcome}${details}`,
  );
}

const byPhase = new Map(phases.map((timing) => [timing.phase, timing.durationMs]));
const setup = byPhase.get("session.create") ?? 0;
const agent = phases
  .filter((timing) => /^agent\.turn\.\d+$/u.test(timing.phase))
  .reduce((total, timing) => total + timing.durationMs, 0);
const grader = byPhase.get("validation.grader") ?? 0;
const checks = Math.max(
  0,
  ...phases
    .filter(
      (timing) => timing.phase.startsWith("validation.") && timing.phase !== "validation.grader",
    )
    .map((timing) => timing.durationMs),
);
console.log(`\nSetup:      ${formatDuration(setup)}`);
console.log(`Agent:      ${formatDuration(agent)}`);
console.log(`Grader:     ${formatDuration(grader)}`);
console.log(`Checks:     ${formatDuration(checks)}`);
console.log(`Validation: ${formatDuration(grader + checks)}`);
console.log(`Total:      ${formatDuration(byPhase.get("run.total") ?? 0)}`);

function resolveTimingsPath(input) {
  const path = resolve(input);
  if (path.endsWith(".json")) return path;
  const candidates = [
    join(path, "project/benchmark/timings.json"),
    join(dirname(path), "project/benchmark/timings.json"),
  ];
  const timingPath = candidates.find(existsSync);
  if (timingPath === undefined) {
    throw new Error(`Could not find project/benchmark/timings.json under ${path}.`);
  }
  return timingPath;
}

function formatDuration(durationMs) {
  return `${(durationMs / 1000).toFixed(1)}s`;
}