Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

6.6 KB
Raw Blame
export const authoringTreatments = ["baseline", "guided"] as const;

export type AuthoringTreatment = (typeof authoringTreatments)[number];

export type AuthoringBenchmarkSupport = "supported" | "candidate" | "superseded";

export interface AuthoringBenchmarkModel {
  readonly id: string;
  readonly model: string;
  /** Model reference sent to the native harness when it differs from the public model ID. */
  readonly agentModel?: string;
  readonly displayName: string;
  readonly harness: "OpenCode" | "Claude Code" | "Codex";
  readonly agentOptions?: Readonly<Record<string, unknown>>;
  readonly support: AuthoringBenchmarkSupport;
}

export const benchmarkModels = [
  {
    id: "claude-sonnet-4-6",
    model: "claude-sonnet-4-6",
    displayName: "Claude Sonnet 4.6",
    harness: "OpenCode",
    support: "superseded",
  },
  {
    id: "kimi-k3",
    model: "moonshotai/kimi-k3",
    displayName: "Kimi K3",
    harness: "OpenCode",
    support: "supported",
  },
  {
    id: "claude-fable-5",
    model: "anthropic/claude-fable-5",
    displayName: "Claude Fable 5",
    harness: "Claude Code",
    support: "superseded",
  },
  {
    id: "claude-fable-5-1",
    model: "anthropic/claude-fable-5.1",
    displayName: "Claude Fable 5.1 (high)",
    harness: "Claude Code",
    agentOptions: { effort: "high" },
    support: "supported",
  },
  {
    id: "grok-4-6",
    model: "xai/grok-4.6",
    displayName: "Grok 4.6",
    harness: "OpenCode",
    support: "superseded",
  },
  {
    id: "gpt-5-6-sol",
    model: "openai/gpt-5.6-sol",
    displayName: "GPT-5.6 Sol",
    harness: "Codex",
    support: "superseded",
  },
  {
    id: "gpt-5-6-terra",
    model: "openai/gpt-5.6-terra",
    displayName: "GPT-5.6 Terra",
    harness: "Codex",
    support: "supported",
  },
  {
    id: "gpt-6-astra-high",
    model: "openai/gpt-6-astra",
    agentModel: "openai/gpt-6-astra?reasoningEffort=high",
    displayName: "GPT-6 Astra (high)",
    harness: "Codex",
    support: "supported",
  },
  {
    id: "claude-sonnet-5",
    model: "anthropic/claude-sonnet-5",
    displayName: "Claude Sonnet 5",
    harness: "Claude Code",
    support: "supported",
  },
  {
    id: "glm-5-2",
    model: "zai/glm-5.2",
    displayName: "GLM 5.2",
    harness: "OpenCode",
    support: "superseded",
  },
  {
    id: "glm-5-3",
    model: "zai/glm-5.3",
    displayName: "GLM 5.3",
    harness: "OpenCode",
    support: "supported",
  },
  {
    id: "claude-opus-5",
    model: "anthropic/claude-opus-5",
    displayName: "Claude Opus 5",
    harness: "Claude Code",
    support: "superseded",
  },
  {
    id: "gemini-3-1-pro-preview",
    model: "google/gemini-3.1-pro-preview",
    displayName: "Gemini 3.1 Pro Preview",
    harness: "OpenCode",
    support: "supported",
  },
  {
    id: "gemini-3-8-flash",
    model: "google/gemini-3.8-flash",
    displayName: "Gemini 3.8 Flash",
    harness: "OpenCode",
    support: "supported",
  },
  {
    id: "minimax-m3",
    model: "minimax/minimax-m3",
    displayName: "MiniMax M3",
    harness: "OpenCode",
    support: "supported",
  },
  {
    id: "kimi-k2-7-code",
    model: "moonshotai/kimi-k2.7-code",
    displayName: "Kimi K2.7 Code",
    harness: "OpenCode",
    support: "supported",
  },
  {
    id: "claude-opus-5-5-high",
    model: "anthropic/claude-opus-5.5",
    displayName: "Claude Opus 5.5 (high)",
    harness: "Claude Code",
    agentOptions: { effort: "high" },
    support: "supported",
  },
  {
    id: "gpt-6-sol-high",
    model: "openai/gpt-6-sol",
    agentModel: "openai/gpt-6-sol?reasoningEffort=high",
    displayName: "GPT-6 Sol (high)",
    harness: "Codex",
    support: "supported",
  },
  {
    id: "gpt-6-luna-high",
    model: "openai/gpt-6-luna",
    agentModel: "openai/gpt-6-luna?reasoningEffort=high",
    displayName: "GPT-6 Luna (high)",
    harness: "Codex",
    support: "supported",
  },
  {
    id: "grok-4-7",
    model: "spacexai/grok-4.7",
    displayName: "Grok 4.7",
    harness: "OpenCode",
    agentOptions: {
      binaryUrl:
        "https://ymdea60kblwwhidh.public.blob.vercel-storage.com/opencode-linux-x64-RCtfS54uaTwa5i9C3bpzguLy3jEBP6",
      extraProviders: {
        vercel: {
          models: {
            "spacexai/grok-4.7": {
              name: "Grok 4.7",
              reasoning: true,
              tool_call: true,
              temperature: true,
              attachment: true,
              modalities: { input: ["text", "image"], output: ["text"] },
              limit: { context: 500000, output: 131072 },
              cost: { input: 0, output: 0, cache_read: 0 },
            },
          },
        },
      },
    },
    support: "supported",
  },
] as const satisfies ReadonlyArray<AuthoringBenchmarkModel>;

export const publishedBenchmarkModels = benchmarkModels.filter(
  (benchmark) => benchmark.support === "supported",
);

export const publishedBenchmark = {
  caseIds: [
    "author-001-weather-tool",
    "author-002-new-project",
    "author-003-openapi-connection",
    "author-004-packaged-skill",
    "author-005-conditional-approval",
    "author-006-custom-channel",
    "author-007-digest-schedule",
  ],
  runs: 3,
} as const;

export function publishedExperimentId(
  benchmark: Pick<AuthoringBenchmarkModel, "id" | "harness">,
  treatment: AuthoringTreatment,
): string {
  return `${benchmark.id}-${harnessId(benchmark.harness)}--${treatment}`;
}

export function harnessId(harness: AuthoringBenchmarkModel["harness"]): string {
  if (harness === "Claude Code") return "claude-code";
  if (harness === "Codex") return "codex";
  return "opencode";
}

export function parseAuthoringTreatment(value: string): AuthoringTreatment {
  if (authoringTreatments.includes(value as AuthoringTreatment)) {
    return value as AuthoringTreatment;
  }
  throw new Error(
    `Unknown treatment ${JSON.stringify(value)}. Expected one of: ${authoringTreatments.join(", ")}.`,
  );
}

export function findBenchmarkModel(value: string): AuthoringBenchmarkModel {
  const benchmark = benchmarkModels.find(
    (candidate) => candidate.id === value || candidate.model === value,
  );
  if (benchmark !== undefined) return benchmark;
  throw new Error(
    `Unknown model ${JSON.stringify(value)}. Expected one of: ${benchmarkModels
      .map((candidate) => candidate.id)
      .join(", ")}.`,
  );
}

export function findPublishedBenchmarkModel(value: string): AuthoringBenchmarkModel {
  const benchmark = publishedBenchmarkModels.find(
    (candidate) => candidate.id === value || candidate.model === value,
  );
  if (benchmark !== undefined) return benchmark;
  throw new Error(
    `Model ${JSON.stringify(value)} is not in the supported publication set. Expected one of: ${publishedBenchmarkModels.map((candidate) => candidate.id).join(", ")}.`,
  );
}