Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

10.7 KB
Raw Blame
// Plans one benchmark run from e2e/benchmark.json:
//
//   live_matrix  attempt × fixture × model legs on the local world, ordered so
//                the legs running together spread across models
//   max_parallel, evals_per_leg
//                the registry's concurrency caps, which bound gateway load
//                and keep evals from sharing a runner
//   mock_matrix  fixture × mock world legs, one attempt each
//   plan.json    the resolved registry snapshot (models pinned to their AI
//                Gateway release, fixture digests, expected eval ids) plus
//                every leg,
//                handed to the publish job
//
// Dispatch inputs narrow the run through BENCHMARK_MODELS and
// BENCHMARK_FIXTURES (comma-separated registry names) and BENCHMARK_ATTEMPTS.
//
// Usage: node scripts/eval-benchmark/plan.mjs <plan.json> [sha]
import { execFileSync } from "node:child_process";
import { appendFileSync, existsSync, readFileSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { fileURLToPath } from "node:url";

import { discoverEvalIds } from "../../.github/scripts/discover-e2e-fixtures.mjs";
import { resolveModelReleases } from "./catalog.mjs";
import { fixtureDigest } from "./digest.mjs";

export const PLAN_SCHEMA_VERSION = 1;
const FIXTURE_ROOTS = ["e2e/fixtures", "apps/fixtures"];
const NAME = /^[a-z0-9-]+$/;
const MAX_ATTEMPTS = 10;
const MAX_PARALLEL_LEGS = 50;
const MAX_EVALS_PER_LEG = 16;

/**
 * Validate the registry and apply dispatch narrowing. Pure apart from the
 * injected fixture lookup, so it is testable without a checkout.
 *
 * @param {{
 *   registry: any,
 *   matrix: any,
 *   findFixture: (name: string) => { dir: string, evalIds: string[], packageJson?: any } | undefined,
 *   narrow?: { models?: string, fixtures?: string, attempts?: string },
 * }} input
 */
export function selectBenchmark({ registry, matrix, findFixture, narrow = {} }) {
  const models = namedEntries(registry?.models, "models");
  const attempts = positiveInteger(registry?.attempts, 'e2e/benchmark.json: "attempts"');
  const concurrency = {
    legs: boundedInteger(
      registry?.concurrency?.legs,
      MAX_PARALLEL_LEGS,
      'e2e/benchmark.json: "concurrency.legs"',
    ),
    evalsPerLeg: boundedInteger(
      registry?.concurrency?.evalsPerLeg,
      MAX_EVALS_PER_LEG,
      'e2e/benchmark.json: "concurrency.evalsPerLeg"',
    ),
  };
  const fixtureNames = names(registry?.fixtures, "fixtures");
  const mockWorlds = names(registry?.mockWorlds ?? [], "mockWorlds", { allowEmpty: true });
  const matrixWorlds = new Map((matrix?.worlds ?? []).map((world) => [world.name, world]));
  for (const world of mockWorlds) {
    // Mock legs boot a local server, so a non-local world must be a package
    // the fixture can select (EVE_E2E_WORKFLOW_WORLD), not a deploy target.
    if (world !== "local" && matrixWorlds.get(world)?.package === undefined) {
      const packaged = [...matrixWorlds.values()].filter((w) => w.package).map((w) => w.name);
      throw new Error(
        `e2e/benchmark.json: mock world "${world}" must be "local" or a world with a package in e2e/matrix.json (${packaged.join(", ")}).`,
      );
    }
  }

  const fixtures = fixtureNames.map((name) => {
    const found = findFixture(name);
    if (found === undefined) {
      throw new Error(
        `e2e/benchmark.json: fixture "${name}" was not found with an evals/ directory under ${FIXTURE_ROOTS.join(" or ")}.`,
      );
    }
    if (found.evalIds.length === 0)
      throw new Error(`e2e/benchmark.json: fixture "${name}" has no *.eval.ts files.`);
    return { name, ...found };
  });

  const selectedModels = narrowNames(models, narrow.models, "model");
  const selectedFixtures = narrowNames(fixtures, narrow.fixtures, "fixture");
  const selectedAttempts =
    narrow.attempts === undefined || narrow.attempts === ""
      ? attempts
      : positiveInteger(Number(narrow.attempts), "attempts input");

  return {
    attempts: selectedAttempts,
    concurrency,
    models: selectedModels,
    fixtures: selectedFixtures,
    mockWorlds: mockWorlds.map((name) => ({ name, package: matrixWorlds.get(name)?.package })),
  };
}

/** Expand a selection into run legs. Leg ids are stable artifact-name suffixes. */
export function planLegs({ models, fixtures, attempts, mockWorlds }) {
  // GitHub starts matrix legs in order and cannot cap concurrency per model,
  // so model-innermost order keeps each wave of legs spread across models and
  // each eval's attempts spread over time.
  const live = [];
  for (let attempt = 1; attempt <= attempts; attempt += 1) {
    for (const fixture of fixtures) {
      for (const model of models) {
        live.push(
          leg({ kind: "live", fixture, model_name: model.name, model_id: model.id, attempt }),
        );
      }
    }
  }
  const mock = [];
  for (const fixture of fixtures) {
    const selectedWorlds = fixture.packageJson?.e2e?.worlds;
    for (const world of mockWorlds) {
      // `local` is the default world every fixture boots on; other worlds
      // follow the fixture's own e2e.worlds selection.
      if (
        world.name !== "local" &&
        Array.isArray(selectedWorlds) &&
        !selectedWorlds.includes(world.name)
      )
        continue;
      mock.push(
        leg({ kind: "mock", fixture, model_name: "mock", model_id: "mock", world, attempt: 1 }),
      );
    }
  }
  return { live, mock };
}

function leg({ kind, fixture, model_name, model_id, world = { name: "local" }, attempt }) {
  return {
    // Registry names never contain "_", so "__" keeps ids unambiguous.
    leg_id: [kind, fixture.name, model_name, world.name, attempt].join("__"),
    kind,
    fixture: fixture.name,
    dir: fixture.dir,
    model_name,
    model_id,
    world: world.name,
    world_package: world.package ?? "",
    attempt,
  };
}

function namedEntries(entries, key) {
  if (!Array.isArray(entries) || entries.length === 0)
    throw new Error(`e2e/benchmark.json: "${key}" must be a non-empty array.`);
  const seen = new Set();
  for (const entry of entries) {
    if (typeof entry?.name !== "string" || !NAME.test(entry.name))
      throw new Error(`e2e/benchmark.json: every "${key}" entry needs a lowercase-dashed "name".`);
    if (typeof entry.id !== "string" || entry.id.length === 0)
      throw new Error(`e2e/benchmark.json: "${key}" entry "${entry.name}" needs a gateway "id".`);
    if (seen.has(entry.name) || seen.has(entry.id))
      throw new Error(`e2e/benchmark.json: duplicate "${key}" entry "${entry.name}".`);
    seen.add(entry.name).add(entry.id);
  }
  return entries.map(({ name, id }) => ({ name, id }));
}

function names(entries, key, { allowEmpty = false } = {}) {
  if (!Array.isArray(entries) || (!allowEmpty && entries.length === 0))
    throw new Error(`e2e/benchmark.json: "${key}" must be a non-empty array.`);
  if (entries.some((entry) => typeof entry !== "string" || !NAME.test(entry)))
    throw new Error(
      `e2e/benchmark.json: "${key}" entries must be lowercase alphanumerics and dashes.`,
    );
  if (new Set(entries).size !== entries.length)
    throw new Error(`e2e/benchmark.json: "${key}" contains duplicates.`);
  return entries;
}

function positiveInteger(value, label) {
  return boundedInteger(value, MAX_ATTEMPTS, label);
}

function boundedInteger(value, max, label) {
  if (!Number.isInteger(value) || value < 1 || value > max)
    throw new Error(`${label} must be an integer from 1 to ${max}.`);
  return value;
}

/** An empty input keeps every entry; otherwise every requested name must exist. */
function narrowNames(entries, input, label) {
  if (input === undefined || input.trim() === "") return entries;
  const requested = input
    .split(",")
    .map((name) => name.trim())
    .filter(Boolean);
  if (requested.length === 1 && requested[0] === "none") return [];
  if (new Set(requested).size !== requested.length)
    throw new Error(`Benchmark ${label} input "${input}" contains duplicates.`);
  const known = new Map(entries.map((entry) => [entry.name, entry]));
  const unknown = requested.filter((name) => !known.has(name));
  if (unknown.length > 0) {
    throw new Error(
      `Unknown benchmark ${label}${unknown.length === 1 ? "" : "s"} ${unknown.join(", ")}; choose from ${[...known.keys()].join(", ")} (or "none").`,
    );
  }
  return requested.map((name) => known.get(name));
}

function findFixtureOnDisk(name) {
  for (const root of FIXTURE_ROOTS) {
    const dir = join(root, name);
    if (!existsSync(join(dir, "evals"))) continue;
    const packageJsonPath = join(dir, "package.json");
    return {
      dir,
      evalIds: discoverEvalIds(join(dir, "evals")),
      packageJson: existsSync(packageJsonPath)
        ? JSON.parse(readFileSync(packageJsonPath, "utf8"))
        : undefined,
    };
  }
  return undefined;
}

if (process.argv[1] === fileURLToPath(import.meta.url)) {
  const [planPath, shaArgument] = process.argv.slice(2);
  if (planPath === undefined) {
    console.error("Usage: node scripts/eval-benchmark/plan.mjs <plan.json> [sha]");
    process.exit(1);
  }
  const sha =
    shaArgument ?? execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }).trim();
  const selection = selectBenchmark({
    registry: JSON.parse(readFileSync("e2e/benchmark.json", "utf8")),
    matrix: JSON.parse(readFileSync("e2e/matrix.json", "utf8")),
    findFixture: findFixtureOnDisk,
    narrow: {
      models: process.env.BENCHMARK_MODELS,
      fixtures: process.env.BENCHMARK_FIXTURES,
      attempts: process.env.BENCHMARK_ATTEMPTS,
    },
  });
  const models = await resolveModelReleases(selection.models);
  const { live, mock } = planLegs({ ...selection, models });
  const plan = {
    schema_version: PLAN_SCHEMA_VERSION,
    sha,
    planned_at: new Date().toISOString(),
    registry: {
      attempts: selection.attempts,
      // Concurrency changes measured latency, so every manifest records it.
      concurrency: {
        legs: selection.concurrency.legs,
        evals_per_leg: selection.concurrency.evalsPerLeg,
      },
      models,
      mock_worlds: selection.mockWorlds.map((world) => world.name),
      fixtures: selection.fixtures.map(({ name, dir, evalIds }) => ({
        name,
        dir,
        digest: fixtureDigest(sha, dir),
        eval_ids: evalIds,
      })),
    },
    legs: [...live, ...mock],
  };
  writeFileSync(planPath, `${JSON.stringify(plan, null, 2)}\n`);
  const lines = [
    `live_matrix=${JSON.stringify(live)}`,
    `mock_matrix=${JSON.stringify(mock)}`,
    `max_parallel=${selection.concurrency.legs}`,
    `evals_per_leg=${selection.concurrency.evalsPerLeg}`,
  ];
  console.error(`Planned ${live.length} live and ${mock.length} mock legs at ${sha}.`);
  if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, `${lines.join("\n")}\n`);
  else process.stdout.write(`${lines.join("\n")}\n`);
}