Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

5.1 KB
Raw Blame
import test from "node:test";
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import { resolveModelReleases } from "./catalog.mjs";
import { planLegs, selectBenchmark } from "./plan.mjs";

const registry = {
  models: [
    { name: "alpha", id: "acme/alpha" },
    { name: "beta", id: "acme/beta" },
  ],
  fixtures: ["agent-one", "agent-two"],
  mockWorlds: ["local", "postgres"],
  attempts: 2,
  concurrency: { legs: 3, evalsPerLeg: 1 },
};
const matrix = {
  worlds: [{ name: "vercel" }, { name: "postgres", package: "@workflow/world-postgres" }],
};
const fixtures = {
  "agent-one": { dir: "e2e/fixtures/agent-one", evalIds: ["a", "b"] },
  "agent-two": {
    dir: "e2e/fixtures/agent-two",
    evalIds: ["c"],
    packageJson: { e2e: { worlds: ["vercel"] } },
  },
};
const select = (overrides = {}, narrow = {}) =>
  selectBenchmark({
    registry: { ...registry, ...overrides },
    matrix,
    findFixture: (name) => fixtures[name],
    narrow,
  });

test("expands fixtures × models × attempts and mock worlds the fixture selects", () => {
  const selection = select();
  assert.deepEqual(selection.concurrency, { legs: 3, evalsPerLeg: 1 });
  const { live, mock } = planLegs(selection);
  assert.equal(live.length, 2 * 2 * 2);
  // Models vary fastest so legs started together spread across models, and
  // each eval's attempts are spread over the run.
  assert.deepEqual(
    live.slice(0, 5).map((leg) => leg.leg_id),
    [
      "live__agent-one__alpha__local__1",
      "live__agent-one__beta__local__1",
      "live__agent-two__alpha__local__1",
      "live__agent-two__beta__local__1",
      "live__agent-one__alpha__local__2",
    ],
  );
  assert.deepEqual(live[0], {
    leg_id: "live__agent-one__alpha__local__1",
    kind: "live",
    fixture: "agent-one",
    dir: "e2e/fixtures/agent-one",
    model_name: "alpha",
    model_id: "acme/alpha",
    world: "local",
    world_package: "",
    attempt: 1,
  });
  assert.equal(new Set(live.map((leg) => leg.leg_id)).size, live.length);
  assert.deepEqual(
    mock.map((leg) => [leg.leg_id, leg.world_package]),
    [
      ["mock__agent-one__mock__local__1", ""],
      ["mock__agent-one__mock__postgres__1", "@workflow/world-postgres"],
      ["mock__agent-two__mock__local__1", ""],
    ],
  );
});

test("dispatch inputs narrow every dimension and reject unknown names", () => {
  const selection = select({}, { models: "beta", fixtures: "agent-two", attempts: "1" });
  assert.deepEqual(
    planLegs(selection).live.map((leg) => leg.leg_id),
    ["live__agent-two__beta__local__1"],
  );
  assert.deepEqual(planLegs(select({}, { fixtures: "none" })), { live: [], mock: [] });
  assert.throws(
    () => select({}, { models: "gamma" }),
    /Unknown benchmark model gamma; choose from alpha, beta/,
  );
  assert.throws(
    () => select({}, { models: "beta,beta" }),
    /Benchmark model input "beta,beta" contains duplicates/,
  );
  assert.throws(
    () => select({}, { attempts: "0" }),
    /attempts input must be an integer from 1 to 10/,
  );
});

test("registry errors name the offending entry", () => {
  for (const [overrides, message] of [
    [{ fixtures: ["agent-missing"] }, /fixture "agent-missing" was not found/],
    [{ mockWorlds: ["moon"] }, /mock world "moon" must be "local" or a world with a package/],
    [{ mockWorlds: ["vercel"] }, /mock world "vercel" must be "local" or a world with a package/],
    [
      {
        models: [
          { name: "alpha", id: "x" },
          { name: "alpha", id: "y" },
        ],
      },
      /duplicate "models" entry "alpha"/,
    ],
    [{ concurrency: undefined }, /"concurrency.legs" must be an integer from 1 to 50/],
    [
      { concurrency: { legs: 5, evalsPerLeg: 0 } },
      /"concurrency.evalsPerLeg" must be an integer from 1 to 16/,
    ],
  ])
    assert.throws(() => select(overrides), message);
});

test("the committed registry is valid", () => {
  const committed = JSON.parse(readFileSync(new URL("../../e2e/benchmark.json", import.meta.url)));
  const committedMatrix = JSON.parse(
    readFileSync(new URL("../../e2e/matrix.json", import.meta.url)),
  );
  const selection = selectBenchmark({
    registry: committed,
    matrix: committedMatrix,
    findFixture: () => ({ dir: "fixture", evalIds: ["eval"] }),
  });
  assert.equal(selection.models.length, committed.models.length);
});

const catalog =
  (data, ok = true) =>
  async () => ({ ok, status: ok ? 200 : 503, json: async () => ({ data }) });

test("pins each model to its catalog release and fails before running on unknown ids", async () => {
  const released = Date.UTC(2026, 8, 22) / 1000;
  const entries = [{ id: "acme/alpha", released }];
  assert.deepEqual(
    await resolveModelReleases([{ name: "alpha", id: "acme/alpha" }], catalog(entries)),
    [{ name: "alpha", id: "acme/alpha", release: "2026-09-22" }],
  );
  await assert.rejects(
    resolveModelReleases([{ name: "beta", id: "acme/beta" }], catalog(entries)),
    /AI Gateway catalog has no model "acme\/beta" \(e2e\/benchmark.json model "beta"\)/,
  );
  await assert.rejects(
    resolveModelReleases([{ name: "alpha", id: "acme/alpha" }], catalog([{ id: "acme/alpha" }])),
    /has no release date to pin/,
  );
});