The Open Framework for Building Agents
export const authoringTreatments = ["baseline", "guided"] as const;
export type AuthoringTreatment = (typeof authoringTreatments)[number];
export type AuthoringBenchmarkSupport = "supported" | "candidate" | "superseded";
export interface AuthoringBenchmarkModel {
readonly id: string;
readonly model: string;
/** Model reference sent to the native harness when it differs from the public model ID. */
readonly agentModel?: string;
readonly displayName: string;
readonly harness: "OpenCode" | "Claude Code" | "Codex";
readonly agentOptions?: Readonly<Record<string, unknown>>;
readonly support: AuthoringBenchmarkSupport;
}
export const benchmarkModels = [
{
id: "claude-sonnet-4-6",
model: "claude-sonnet-4-6",
displayName: "Claude Sonnet 4.6",
harness: "OpenCode",
support: "superseded",
},
{
id: "kimi-k3",
model: "moonshotai/kimi-k3",
displayName: "Kimi K3",
harness: "OpenCode",
support: "supported",
},
{
id: "claude-fable-5",
model: "anthropic/claude-fable-5",
displayName: "Claude Fable 5",
harness: "Claude Code",
support: "superseded",
},
{
id: "claude-fable-5-1",
model: "anthropic/claude-fable-5.1",
displayName: "Claude Fable 5.1 (high)",
harness: "Claude Code",
agentOptions: { effort: "high" },
support: "supported",
},
{
id: "grok-4-6",
model: "xai/grok-4.6",
displayName: "Grok 4.6",
harness: "OpenCode",
support: "superseded",
},
{
id: "gpt-5-6-sol",
model: "openai/gpt-5.6-sol",
displayName: "GPT-5.6 Sol",
harness: "Codex",
support: "superseded",
},
{
id: "gpt-5-6-terra",
model: "openai/gpt-5.6-terra",
displayName: "GPT-5.6 Terra",
harness: "Codex",
support: "supported",
},
{
id: "gpt-6-astra-high",
model: "openai/gpt-6-astra",
agentModel: "openai/gpt-6-astra?reasoningEffort=high",
displayName: "GPT-6 Astra (high)",
harness: "Codex",
support: "supported",
},
{
id: "claude-sonnet-5",
model: "anthropic/claude-sonnet-5",
displayName: "Claude Sonnet 5",
harness: "Claude Code",
support: "supported",
},
{
id: "glm-5-2",
model: "zai/glm-5.2",
displayName: "GLM 5.2",
harness: "OpenCode",
support: "superseded",
},
{
id: "glm-5-3",
model: "zai/glm-5.3",
displayName: "GLM 5.3",
harness: "OpenCode",
support: "supported",
},
{
id: "claude-opus-5",
model: "anthropic/claude-opus-5",
displayName: "Claude Opus 5",
harness: "Claude Code",
support: "superseded",
},
{
id: "gemini-3-1-pro-preview",
model: "google/gemini-3.1-pro-preview",
displayName: "Gemini 3.1 Pro Preview",
harness: "OpenCode",
support: "supported",
},
{
id: "gemini-3-8-flash",
model: "google/gemini-3.8-flash",
displayName: "Gemini 3.8 Flash",
harness: "OpenCode",
support: "supported",
},
{
id: "minimax-m3",
model: "minimax/minimax-m3",
displayName: "MiniMax M3",
harness: "OpenCode",
support: "supported",
},
{
id: "kimi-k2-7-code",
model: "moonshotai/kimi-k2.7-code",
displayName: "Kimi K2.7 Code",
harness: "OpenCode",
support: "supported",
},
{
id: "claude-opus-5-5-high",
model: "anthropic/claude-opus-5.5",
displayName: "Claude Opus 5.5 (high)",
harness: "Claude Code",
agentOptions: { effort: "high" },
support: "supported",
},
{
id: "gpt-6-sol-high",
model: "openai/gpt-6-sol",
agentModel: "openai/gpt-6-sol?reasoningEffort=high",
displayName: "GPT-6 Sol (high)",
harness: "Codex",
support: "supported",
},
{
id: "gpt-6-luna-high",
model: "openai/gpt-6-luna",
agentModel: "openai/gpt-6-luna?reasoningEffort=high",
displayName: "GPT-6 Luna (high)",
harness: "Codex",
support: "supported",
},
{
id: "grok-4-7",
model: "spacexai/grok-4.7",
displayName: "Grok 4.7",
harness: "OpenCode",
agentOptions: {
binaryUrl:
"https://ymdea60kblwwhidh.public.blob.vercel-storage.com/opencode-linux-x64-RCtfS54uaTwa5i9C3bpzguLy3jEBP6",
extraProviders: {
vercel: {
models: {
"spacexai/grok-4.7": {
name: "Grok 4.7",
reasoning: true,
tool_call: true,
temperature: true,
attachment: true,
modalities: { input: ["text", "image"], output: ["text"] },
limit: { context: 500000, output: 131072 },
cost: { input: 0, output: 0, cache_read: 0 },
},
},
},
},
},
support: "supported",
},
] as const satisfies ReadonlyArray<AuthoringBenchmarkModel>;
export const publishedBenchmarkModels = benchmarkModels.filter(
(benchmark) => benchmark.support === "supported",
);
export const publishedBenchmark = {
caseIds: [
"author-001-weather-tool",
"author-002-new-project",
"author-003-openapi-connection",
"author-004-packaged-skill",
"author-005-conditional-approval",
"author-006-custom-channel",
"author-007-digest-schedule",
],
runs: 3,
} as const;
export function publishedExperimentId(
benchmark: Pick<AuthoringBenchmarkModel, "id" | "harness">,
treatment: AuthoringTreatment,
): string {
return `${benchmark.id}-${harnessId(benchmark.harness)}--${treatment}`;
}
export function harnessId(harness: AuthoringBenchmarkModel["harness"]): string {
if (harness === "Claude Code") return "claude-code";
if (harness === "Codex") return "codex";
return "opencode";
}
export function parseAuthoringTreatment(value: string): AuthoringTreatment {
if (authoringTreatments.includes(value as AuthoringTreatment)) {
return value as AuthoringTreatment;
}
throw new Error(
`Unknown treatment ${JSON.stringify(value)}. Expected one of: ${authoringTreatments.join(", ")}.`,
);
}
export function findBenchmarkModel(value: string): AuthoringBenchmarkModel {
const benchmark = benchmarkModels.find(
(candidate) => candidate.id === value || candidate.model === value,
);
if (benchmark !== undefined) return benchmark;
throw new Error(
`Unknown model ${JSON.stringify(value)}. Expected one of: ${benchmarkModels
.map((candidate) => candidate.id)
.join(", ")}.`,
);
}
export function findPublishedBenchmarkModel(value: string): AuthoringBenchmarkModel {
const benchmark = publishedBenchmarkModels.find(
(candidate) => candidate.id === value || candidate.model === value,
);
if (benchmark !== undefined) return benchmark;
throw new Error(
`Model ${JSON.stringify(value)} is not in the supported publication set. Expected one of: ${publishedBenchmarkModels.map((candidate) => candidate.id).join(", ")}.`,
);
}