Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

1.8 KB
Raw Blame
import type { ExperimentConfig } from "@vercel/agent-eval";

import {
  harnessId,
  type AuthoringBenchmarkModel,
  type AuthoringTreatment,
  publishedBenchmark,
} from "./benchmark-config.js";
import { createNativeAuthoringSetup } from "./native-authoring-setup.js";

export function authoringExperiment(options: {
  readonly revision: string;
  readonly packageSpec: string;
  readonly runs?: number;
  readonly evals?: readonly string[];
  readonly benchmark: AuthoringBenchmarkModel;
  readonly treatment: AuthoringTreatment;
  readonly verbose?: boolean;
}): ExperimentConfig {
  return {
    agent: `vercel-ai-gateway/${harnessId(options.benchmark.harness)}`,
    model: nativeModel(options.benchmark),
    evals:
      process.env.EVE_BENCHMARK_EVAL ??
      (options.evals ? [...options.evals] : [...publishedBenchmark.caseIds]),
    scripts: ["typecheck", "build"],
    runs: options.runs ?? 1,
    earlyExit: false,
    // Lower this when iterating on a case that stalls, so a hung turn surfaces
    // in minutes instead of consuming the full budget.
    timeout: Number(process.env.EVE_BENCHMARK_TIMEOUT ?? 900),
    sandbox: "vercel",
    copyFiles: "changed",
    setup: createNativeAuthoringSetup({
      packageSpec: options.packageSpec,
      revision: options.revision,
      treatment: options.treatment,
    }),
    agentOptions: {
      cliPackage:
        options.benchmark.harness === "Claude Code"
          ? "@anthropic-ai/[email protected]"
          : undefined,
      ...options.benchmark.agentOptions,
      verbose: options.verbose ?? false,
    },
  };
}

function nativeModel(benchmark: AuthoringBenchmarkModel): string {
  const model = benchmark.agentModel ?? benchmark.model;
  if (benchmark.harness !== "Claude Code") return model;
  return model.replace(/^anthropic\//u, "");
}