Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

automation script to pull models.yml (#9635)

Shahmir Varqha9a9c8cc9aa5d
diff --git a/frontend/src/components/ai/__tests__/ai-utils.test.ts b/frontend/src/components/ai/__tests__/ai-utils.test.ts
index fdfd314052..217f9347ea 100644
--- a/frontend/src/components/ai/__tests__/ai-utils.test.ts
+++ b/frontend/src/components/ai/__tests__/ai-utils.test.ts
@@ -4,46 +4,51 @@ import type { AiModel } from "@marimo-team/llm-info";
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { UserConfig } from "@/core/config/config-schema";
-// Mock the models.json import
vi.mock("@marimo-team/llm-info/models.json", () => {
- const models: AiModel[] = [
- {
- name: "GPT-4",
- model: "gpt-4",
- description: "OpenAI GPT-4 model",
- providers: ["openai"],
- roles: ["chat", "edit"],
- thinking: false,
- },
- {
- name: "Claude 3",
- model: "claude-3-sonnet",
- description: "Anthropic Claude 3 Sonnet",
- providers: ["anthropic"],
- roles: ["chat", "edit"],
- thinking: false,
- },
- {
- name: "Gemini Pro",
- model: "gemini-pro",
- description: "Google Gemini Pro model",
- providers: ["google"],
- roles: ["chat", "edit"],
- thinking: false,
- },
- {
- name: "Ollama Model",
- model: "llama2",
- description: "Ollama Llama 2 model",
- providers: ["ollama"],
- roles: ["chat", "edit"],
- thinking: false,
- },
- ];
-
- return {
- models: models,
+ const make = (
+ overrides: Partial<AiModel> & Pick<AiModel, "name" | "model">,
+ ): AiModel => ({
+ description: "",
+ roles: ["chat", "edit"],
+ capabilities: [],
+ input_types: [],
+ output_types: [],
+ release_date: "1970-01-01",
+ ...overrides,
+ });
+
+ const models: Record<string, AiModel[]> = {
+ openai: [
+ make({
+ name: "GPT-4",
+ model: "gpt-4",
+ description: "OpenAI GPT-4 model",
+ }),
+ ],
+ anthropic: [
+ make({
+ name: "Claude 3",
+ model: "claude-3-sonnet",
+ description: "Anthropic Claude 3 Sonnet",
+ }),
+ ],
+ google: [
+ make({
+ name: "Gemini Pro",
+ model: "gemini-pro",
+ description: "Google Gemini Pro model",
+ }),
+ ],
+ ollama: [
+ make({
+ name: "Ollama Model",
+ model: "llama2",
+ description: "Ollama Llama 2 model",
+ }),
+ ],
};
+
+ return { models };
});
// Must import after mock
diff --git a/frontend/src/components/ai/ai-model-dropdown.tsx b/frontend/src/components/ai/ai-model-dropdown.tsx
index 8b639a7e75..179dcf4c36 100644
--- a/frontend/src/components/ai/ai-model-dropdown.tsx
+++ b/frontend/src/components/ai/ai-model-dropdown.tsx
@@ -305,7 +305,7 @@ const AiModelDropdownItem = ({
<div className="flex flex-row w-full items-center">
<span>{model.name}</span>
<div className="ml-auto">
- {model.thinking && (
+ {model.capabilities.includes("thinking") && (
<Tooltip content="Reasoning model">
<BrainIcon
className={`h-5 w-5 rounded-md p-1 ${getTagColour("thinking")}`}
@@ -362,7 +362,7 @@ export const AiModelInfoDisplay = ({
</div>
)}
- {model.thinking && (
+ {model.capabilities.includes("thinking") && (
<div className="flex items-center gap-2">
<div className="w-2 h-2 bg-purple-500 rounded-full animate-pulse" />
<span className="text-xs text-muted-foreground">
diff --git a/frontend/src/components/app-config/ai-config.tsx b/frontend/src/components/app-config/ai-config.tsx
index 6f20c08693..0e22a4b46f 100644
--- a/frontend/src/components/app-config/ai-config.tsx
+++ b/frontend/src/components/app-config/ai-config.tsx
@@ -509,7 +509,7 @@ const ModelInfoCard = ({ model }: { model: AiModel }) => {
<Tooltip content="Custom model">
{model.custom && <BotIcon className="h-4 w-4" />}
</Tooltip>
- {model.thinking && (
+ {model.capabilities.includes("thinking") && (
<div
className={cn(
"flex items-center gap-1 rounded px-1 py-0.5 w-fit",
diff --git a/frontend/src/core/ai/__tests__/model-registry.test.ts b/frontend/src/core/ai/__tests__/model-registry.test.ts
index a4053b99de..8eaa82f055 100644
--- a/frontend/src/core/ai/__tests__/model-registry.test.ts
+++ b/frontend/src/core/ai/__tests__/model-registry.test.ts
@@ -1,46 +1,52 @@
/* Copyright 2026 Marimo. All rights reserved. */
import { beforeEach, describe, expect, it, vi } from "vitest";
-// Mock the models.json import
vi.mock("@marimo-team/llm-info/models.json", () => {
- const models: AiModel[] = [
- {
- name: "GPT-4",
- model: "gpt-4",
- description: "OpenAI GPT-4 model",
- providers: ["openai"],
- roles: ["chat", "edit"],
- thinking: false,
- },
- {
- name: "Claude 3",
- model: "claude-3-sonnet",
- description: "Anthropic Claude 3 Sonnet",
- providers: ["anthropic"],
- roles: ["chat", "edit"],
- thinking: false,
- },
- {
- name: "Gemini Pro",
- model: "gemini-pro",
- description: "Google Gemini Pro model",
- providers: ["google"],
- roles: ["chat", "edit"],
- thinking: false,
- },
- {
- name: "Multi Provider Model",
- model: "multi-model",
- description: "Model available on multiple providers",
- providers: ["openai", "anthropic"],
- roles: ["chat", "edit"],
- thinking: false,
- },
- ];
-
- return {
- models: models,
+ const make = (
+ overrides: Partial<AiModel> & Pick<AiModel, "name" | "model">,
+ ): AiModel => ({
+ description: "",
+ roles: ["chat", "edit"],
+ capabilities: [],
+ input_types: [],
+ output_types: [],
+ release_date: "1970-01-01",
+ ...overrides,
+ });
+
+ const multiModel = make({
+ name: "Multi Provider Model",
+ model: "multi-model",
+ description: "Model available on multiple providers",
+ });
+
+ const models: Record<string, AiModel[]> = {
+ openai: [
+ make({
+ name: "GPT-4",
+ model: "gpt-4",
+ description: "OpenAI GPT-4 model",
+ }),
+ multiModel,
+ ],
+ anthropic: [
+ make({
+ name: "Claude 3",
+ model: "claude-3-sonnet",
+ description: "Anthropic Claude 3 Sonnet",
+ }),
+ multiModel,
+ ],
+ google: [
+ make({
+ name: "Gemini Pro",
+ model: "gemini-pro",
+ description: "Google Gemini Pro model",
+ }),
+ ],
};
+
+ return { models };
});
import type { AiModel } from "@marimo-team/llm-info";
@@ -107,14 +113,15 @@ describe("AiModelRegistry", () => {
});
const ids = [...registry.getModelsMap().keys()];
- // Include custom and all default ones.
+ // Include custom and all default ones; iteration follows provider
+ // sections in the source data (openai → anthropic → google).
expect(ids).toEqual([
"openai/custom-gpt",
"openai/gpt-4",
- "anthropic/claude-3-sonnet",
- "google/gemini-pro",
"openai/multi-model",
+ "anthropic/claude-3-sonnet",
"anthropic/multi-model",
+ "google/gemini-pro",
]);
});
});
@@ -125,9 +132,9 @@ describe("AiModelRegistry", () => {
const openaiModels = registry.getModelsByProvider("openai");
expect(openaiModels).toHaveLength(2); // gpt-4 and multi-model
- expect(
- openaiModels.every((model) => model.providers.includes("openai")),
- ).toBe(true);
+ expect(openaiModels.every((model) => model.provider === "openai")).toBe(
+ true,
+ );
});
it("should return empty array for provider with no models", () => {
@@ -147,9 +154,9 @@ describe("AiModelRegistry", () => {
expect(customModel?.name).toBe("custom-gpt");
expect(customModel?.model).toBe("custom-gpt");
expect(customModel?.description).toBe("Custom model");
- expect(customModel?.providers).toEqual(["openai"]);
+ expect(customModel?.provider).toBe("openai");
expect(customModel?.roles).toEqual([]);
- expect(customModel?.thinking).toBe(false);
+ expect(customModel?.capabilities).toEqual([]);
});
it("should filter models based on displayed models", () => {
@@ -309,7 +316,10 @@ describe("AiModelRegistry", () => {
expect(multiModelInOpenai).toBeDefined();
expect(multiModelInAnthropic).toBeDefined();
- expect(multiModelInOpenai).toEqual(multiModelInAnthropic);
+ // Same model id, but each entry belongs to its own provider.
+ expect(multiModelInOpenai?.provider).toBe("openai");
+ expect(multiModelInAnthropic?.provider).toBe("anthropic");
+ expect(multiModelInOpenai?.name).toBe(multiModelInAnthropic?.name);
});
it("should handle displayed models filter with non-existent models", () => {
@@ -339,20 +349,20 @@ describe("AiModelRegistry", () => {
expect(model).toHaveProperty("name");
expect(model).toHaveProperty("model");
expect(model).toHaveProperty("description");
- expect(model).toHaveProperty("providers");
+ expect(model).toHaveProperty("provider");
expect(model).toHaveProperty("roles");
- expect(model).toHaveProperty("thinking");
+ expect(model).toHaveProperty("capabilities");
expect(model).toHaveProperty("custom");
expect(typeof model.name).toBe("string");
expect(typeof model.model).toBe("string");
expect(typeof model.description).toBe("string");
- expect(Array.isArray(model.providers)).toBe(true);
+ expect(typeof model.provider).toBe("string");
expect(Array.isArray(model.roles)).toBe(true);
- expect(typeof model.thinking).toBe("boolean");
+ expect(Array.isArray(model.capabilities)).toBe(true);
expect(typeof model.custom).toBe("boolean");
- expect(model.providers).toContain(provider);
+ expect(model.provider).toBe(provider);
}
}
});
@@ -367,31 +377,33 @@ describe("AiModelRegistry", () => {
expect(customModel).toMatchInlineSnapshot(`
{
+ "capabilities": [],
"custom": true,
"description": "Custom model",
+ "input_types": [],
"model": "custom-gpt",
"name": "custom-gpt",
- "providers": [
- "openai",
- ],
+ "output_types": [],
+ "provider": "openai",
+ "release_date": "1970-01-01",
"roles": [],
- "thinking": false,
}
`);
expect(defaultModel).toMatchInlineSnapshot(`
{
+ "capabilities": [],
"custom": false,
"description": "OpenAI GPT-4 model",
+ "input_types": [],
"model": "gpt-4",
"name": "GPT-4",
- "providers": [
- "openai",
- ],
+ "output_types": [],
+ "provider": "openai",
+ "release_date": "1970-01-01",
"roles": [
"chat",
"edit",
],
- "thinking": false,
}
`);
});
diff --git a/frontend/src/core/ai/model-registry.ts b/frontend/src/core/ai/model-registry.ts
index d1c57f2887..aee0032804 100644
--- a/frontend/src/core/ai/model-registry.ts
+++ b/frontend/src/core/ai/model-registry.ts
@@ -1,12 +1,8 @@
/* Copyright 2026 Marimo. All rights reserved. */
-import type {
- AiModel as AiModelType,
- AiProvider,
- Role,
-} from "@marimo-team/llm-info";
-import { models } from "@marimo-team/llm-info/models.json";
-import { providers } from "@marimo-team/llm-info/providers.json";
+import type { AiModel as AiModelType, AiProvider } from "@marimo-team/llm-info";
+import { models as modelsJson } from "@marimo-team/llm-info/models.json";
+import { providers as providersJson } from "@marimo-team/llm-info/providers.json";
import { Logger } from "@/utils/Logger";
import { MultiMap } from "@/utils/multi-map";
import { once } from "@/utils/once";
@@ -14,13 +10,19 @@ import type { ProviderId } from "./ids/ids";
import { AiModelId, type QualifiedModelId, type ShortModelId } from "./ids/ids";
export interface AiModel extends AiModelType {
- roles: Role[];
model: ShortModelId;
- providers: ProviderId[];
+ /** The provider this entry belongs to. */
+ provider: ProviderId;
/** Whether this is a custom model. */
custom: boolean;
}
+// JSON shape matches the `AiModel` schema (Zod-validated at codegen time).
+const models = modelsJson as unknown as Partial<
+ Record<ProviderId, AiModelType[]>
+>;
+const providers = providersJson as unknown as readonly AiProvider[];
+
interface KnownModelMaps {
/** Map of qualified model ID to model info */
modelMap: ReadonlyMap<QualifiedModelId, AiModel>;
@@ -32,24 +34,25 @@ export const getKnownModelMaps = once((): KnownModelMaps => {
const modelMap = new Map<QualifiedModelId, AiModel>();
const defaultModelByProvider = new Map<ProviderId, QualifiedModelId>();
- for (const model of models) {
- const modelId = model.model as ShortModelId;
- const modelInfo: AiModel = {
- ...model,
- model: model.model as ShortModelId,
- roles: model.roles.map((role) => role as Role),
- providers: model.providers as ProviderId[],
- custom: false,
- };
-
- const supportsChatOrEdit =
- modelInfo.roles.includes("chat") || modelInfo.roles.includes("edit");
+ for (const [providerKey, providerModels] of Object.entries(models)) {
+ if (!providerModels) {
+ continue;
+ }
+ const provider = providerKey as ProviderId;
+ for (const raw of providerModels) {
+ const modelId = raw.model as ShortModelId;
+ const modelInfo: AiModel = {
+ ...raw,
+ model: modelId,
+ provider,
+ custom: false,
+ };
- for (const provider of modelInfo.providers) {
const qualifiedModelId: QualifiedModelId = `${provider}/${modelId}`;
modelMap.set(qualifiedModelId, modelInfo);
- // Track first model per provider that supports chat or edit
+ const supportsChatOrEdit =
+ modelInfo.roles.includes("chat") || modelInfo.roles.includes("edit");
if (supportsChatOrEdit && !defaultModelByProvider.has(provider)) {
defaultModelByProvider.set(provider, qualifiedModelId);
}
@@ -67,9 +70,8 @@ const getProviderMap = once(
const providerMap = new Map<ProviderId, AiProvider>();
const providerToOrderIdx = new Map<ProviderId, number>();
providers.forEach((provider, idx) => {
- const providerId = provider.id as ProviderId;
- providerMap.set(providerId, provider);
- providerToOrderIdx.set(providerId, idx);
+ providerMap.set(provider.id, provider);
+ providerToOrderIdx.set(provider.id, idx);
});
return { providerMap, providerToOrderIdx };
},
@@ -158,9 +160,12 @@ export class AiModelRegistry {
name: modelId.shortModelId,
model: modelId.shortModelId,
description: "Custom model",
- providers: [modelId.providerId],
+ provider: modelId.providerId,
roles: [],
- thinking: false,
+ capabilities: [],
+ input_types: [],
+ output_types: [],
+ release_date: "1970-01-01",
custom: true,
};
customModelsMap.set(model, modelInfo);
diff --git a/packages/llm-info/README.md b/packages/llm-info/README.md
index 8f670d873c..40b5753229 100644
--- a/packages/llm-info/README.md
+++ b/packages/llm-info/README.md
@@ -12,4 +12,14 @@ If you want to add a new LLM model or provider, you can do so by editing the YAM
> **Note:**
> To make it easier for users to choose, keep the number of models to a minimum. Focus on including the latest or recommended models from each provider.
-> Review https://openrouter.ai/provider/{provider_name} for the latest models and capabilities.
+
+## Syncing from `models.dev`
+
+`pnpm sync-models` appends latest models from [`models.dev`](https://models.dev/api.json) to `models.yml`. Existing entries are preserved. Run `pnpm codegen` afterwards.
+
+```bash
+pnpm sync-models # all providers, 10 newest each
+pnpm sync-models --provider=anthropic # one provider
+pnpm sync-models -p openai,google -n 5 # multiple providers, 5 each
+pnpm sync-models --replace # destructive rebuild
+```
diff --git a/packages/llm-info/data/models.yml b/packages/llm-info/data/models.yml
index 0f5d13bdf1..f01d97c9f2 100644
--- a/packages/llm-info/data/models.yml
+++ b/packages/llm-info/data/models.yml
@@ -1,481 +1,755 @@
-# Information provided by https://openrouter.ai/
-# Keep latest models at the top, deprecate old models
-
-# Anthropic
-
-- name: Claude Opus 4.7
- model: claude-opus-4-7
- description: Latest Opus model, strongest for coding and long-running professional tasks
- providers: [anthropic]
- roles: [chat, edit]
- thinking: true
-
-- name: Claude Sonnet 4.6
- model: claude-sonnet-4-6
- description: Most capable Sonnet-class model, with frontier performance across coding, agents, and professional work
- providers: [anthropic]
- roles: [chat, edit]
- thinking: true
-
-- name: Claude Opus 4.6
- model: claude-opus-4-6
- description: Strongest model for coding and long-running professional tasks
- providers: [anthropic]
- roles: [chat, edit]
- thinking: true
-
-- name: Claude Opus 4.5
- model: claude-opus-4-5
- description: Frontier reasoning model optimized for complex software engineering
- providers: [anthropic]
- roles: [chat, edit]
- thinking: true
-
-- name: Claude Sonnet 4.5
- model: claude-sonnet-4-5
- description: Advanced Sonnet model optimized for real-world agents and coding workflows
- providers: [anthropic]
- roles: [chat, edit]
- thinking: true
-
-- name: Claude Haiku 4.5
- model: claude-haiku-4-5
- description: Most efficient model, delivering near-frontier intelligence at a fraction of the cost
- providers: [anthropic]
- roles: [chat, edit]
- thinking: true
-
-# Google
-
-- name: Gemini 3.1 Flash Lite Preview
- model: gemini-3.1-flash-lite-preview
- description: High-efficiency model optimized for high-volume use cases
- providers: [google]
- roles: [chat, edit]
- thinking: false
-
-- name: Gemini 3.1 Pro Preview
- model: gemini-3.1-pro-preview
- description: Frontier reasoning model with enhanced software engineering and agentic reliability
- providers: [google]
- roles: [chat, edit]
- thinking: true
-
-- name: Gemini 3 Flash Preview
- model: gemini-3-flash-preview
- description: High-speed thinking model for agentic workflows, multi-turn chat, and coding assistance
- providers: [google]
- roles: [chat, edit]
- thinking: true
-
-- name: Gemini 2.5 Flash
- model: gemini-2.5-flash
- description: State-of-the-art workhorse model with built-in thinking capabilities
- providers: [google]
- roles: [chat, edit]
- thinking: true
-
-- name: Gemini 2.5 Flash Lite
- model: gemini-2.5-flash-lite
- description: Most cost-efficient model supporting high throughput
- providers: [google]
- roles: [chat, edit]
- thinking: false
-
-# OpenAI
-
-- name: GPT-5.5
- model: gpt-5.5
- description: Newest frontier model for the most complex professional work
- providers: [openai]
- roles: [chat, edit]
- thinking: true
-
-- name: GPT-5.4
- model: gpt-5.4
- description: The latest frontier-grade model, unifying the Codex and GPT lines into a single system
- providers: [openai]
- roles: [chat, edit]
- thinking: false
-
-- name: GPT-5.3 Chat
- model: gpt-5.3-chat
- description: Most-used model that makes everyday conversations smoother and more useful
- providers: [openai, azure]
- roles: [chat, edit]
- thinking: false
-
-- name: GPT-5.3-Codex
- model: gpt-5.3-codex
- description: Most advanced agentic coding model combining frontier software engineering with broad reasoning
- providers: [openai]
- roles: [chat, edit]
- thinking: false
-
-- name: GPT-5 Mini
- model: gpt-5-mini
- description: A faster, cost-efficient version of GPT-5 for well-defined tasks
- providers: [openai, azure, github]
- roles: [chat, edit]
- thinking: false
-
-- name: GPT-5 Nano
- model: gpt-5-nano
- description: Fastest, most cost-efficient version of GPT-5
- providers: [openai, azure]
- roles: [chat, edit]
- thinking: false
-
-- name: o3
- model: o3
- description: Most powerful reasoning model with chain-of-thought capabilities
- providers: [openai, azure]
- roles: [chat, edit]
- thinking: true
-
-- name: o4 Mini
- model: o4-mini
- description: Faster, more affordable reasoning model optimized for math, coding, and visual tasks
- providers: [openai, azure]
- roles: [chat, edit]
- thinking: true
-
-# GitHub
-
-- name: GPT-3.5-turbo
- model: gpt-3.5-turbo
- description: Fast and efficient model with excellent performance for everyday tasks
- providers: [github]
- roles: [chat, edit]
- thinking: false
-
-# Bedrock
-# To verify/update Bedrock model IDs, see https://github.com/marimo-team/marimo/issues/9280
-
-- name: Claude 4.7 Opus (Bedrock)
- model: global.anthropic.claude-opus-4-7
- description: Latest Opus model, strongest for coding and long-running professional tasks
- providers: [bedrock]
- roles: [chat, edit]
- thinking: true
-
-- name: Claude 4.6 Sonnet (Bedrock)
- model: global.anthropic.claude-sonnet-4-6
- description: Most capable Sonnet-class model, with frontier performance across coding, agents, and professional work
- providers: [bedrock]
- roles: [chat, edit]
- thinking: true
-
-- name: Claude 4.6 Opus (Bedrock)
- model: global.anthropic.claude-opus-4-6-v1
- description: Strongest model for coding and long-running professional tasks
- providers: [bedrock]
- roles: [chat, edit]
- thinking: true
-
-- name: Claude 4.5 Opus (Bedrock)
- model: global.anthropic.claude-opus-4-5-20251101-v1:0
- description: Frontier reasoning model optimized for complex software engineering
- providers: [bedrock]
- roles: [chat, edit]
- thinking: true
-
-- name: Claude 4.5 Sonnet (Bedrock)
- model: global.anthropic.claude-sonnet-4-5-20250929-v1:0
- description: Most advanced Sonnet model to date, optimized for real-world agents
- providers: [bedrock]
- roles: [chat, edit]
- thinking: true
-
-- name: Claude 4.5 Haiku (Bedrock)
- model: global.anthropic.claude-haiku-4-5-20251001-v1:0
- description: Most efficient model, delivering near-frontier intelligence at a fraction of the cost
- providers: [bedrock]
- roles: [chat, edit]
- thinking: true
-
-- name: Llama 3.3 70B Instruct
- model: us.meta.llama3-3-70b-instruct-v1:0
- description: Large language model optimized for instruction following and chat
- providers: [bedrock]
- roles: [chat, edit]
- thinking: false
-
-- name: Cohere Embed Multilingual v3
- model: cohere.embed-multilingual-v3
- description: Multilingual text embedding model for semantic search and similarity
- providers: [bedrock]
- roles: [embed]
- thinking: false
-
-- name: Cohere Rerank 3.5
- model: cohere.rerank-v3-5:0
- description: Advanced reranking model for improving search result relevance
- providers: [bedrock]
- roles: [rerank]
- thinking: false
-
-# Ollama Models
-
-- name: gpt-oss 20b
- model: gpt-oss:20b
- description: OpenAI's open-weight models designed for powerful reasoning, agentic tasks, and versatile developer use cases.
- providers: [ollama]
- roles: [chat, edit]
- thinking: true
-
-- name: Qwen3 8b
- model: qwen3:8b
- description: Qwen3 is the latest generation of large language models in Qwen series.
- providers: [ollama]
- roles: [chat, edit]
- thinking: true
-
-- name: Gemma3 12b
- model: gemma3:12b
- description: Lightweight gemma models trained by Google, trained to run on a single GPU.
- providers: [ollama]
- roles: [chat, edit]
- thinking: true
-
-# Weights & Biases Models
-
-- name: DeepSeek V3.1
- model: deepseek-ai/DeepSeek-V3.1
- description: Large hybrid Mixture-of-Experts model that supports both thinking and non-thinking modes
- providers: [wandb]
- roles: [chat, edit]
- thinking: true
-
-- name: Google Gemma 4 31B
- model: google/gemma-4-31B-it
- description: Vision and text model designed for advanced reasoning, agentic workflows, and longer context
- providers: [wandb]
- roles: [chat, edit]
- thinking: true
-
-- name: IBM Granite 4.1 8B
- model: ibm-granite/granite-4.1-8b
- description: Long-context text model with enhanced tool calling and instruction following
- providers: [wandb]
- roles: [chat, edit]
- thinking: false
-
-- name: Meta Llama 3.3 70B (W&B)
- model: meta-llama/Llama-3.3-70B-Instruct
- description: Multilingual model excelling in conversational tasks, detailed instruction-following, and coding
- providers: [wandb]
- roles: [chat, edit]
- thinking: false
-
-- name: Meta Llama 3.1 70B (W&B)
- model: meta-llama/Llama-3.1-70B-Instruct
- description: Multilingual chatbot model optimized for responsiveness
- providers: [wandb]
- roles: [chat, edit]
- thinking: false
-
-- name: Meta Llama 3.1 8B (W&B)
- model: meta-llama/Llama-3.1-8B-Instruct
- description: Efficient conversational model optimized for responsive multilingual chatbot interactions
- providers: [wandb]
- roles: [chat, edit]
- thinking: false
-
-- name: Microsoft Phi 4 Mini 3.8B
- model: microsoft/Phi-4-mini-instruct
- description: Compact instruction-tuned model for resource-constrained environments
- providers: [wandb]
- roles: [chat, edit]
- thinking: false
-
-- name: MiniMax M2.5
- model: MiniMaxAI/MiniMax-M2.5
- description: Mixture-of-Experts model with a highly sparse architecture designed for high-throughput and low latency
- providers: [wandb]
- roles: [chat, edit]
- thinking: true
-
-- name: Moonshot AI Kimi K2.6
- model: moonshotai/Kimi-K2.6
- description: Multimodal Mixture-of-Experts model with 32B activated parameters and 1T total parameters
- providers: [wandb]
- roles: [chat, edit]
- thinking: true
-
-- name: Moonshot AI Kimi K2.5
- model: moonshotai/Kimi-K2.5
- description: Multimodal Mixture-of-Experts model optimized for complex tool use, reasoning, and code synthesis
- providers: [wandb]
- roles: [chat, edit]
- thinking: true
-
-- name: NVIDIA Nemotron 3 Super 120B
- model: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8
- description: LatentMoE model for agentic, reasoning, and conversational capabilities
- providers: [wandb]
- roles: [chat, edit]
- thinking: true
-
-- name: OpenAI GPT OSS 120B
- model: openai/gpt-oss-120b
- description: Efficient Mixture-of-Experts model designed for high-reasoning, agentic and general-purpose use cases
- providers: [wandb]
- roles: [chat, edit]
- thinking: true
-
-- name: OpenAI GPT OSS 20B
- model: openai/gpt-oss-20b
- description: Lower latency Mixture-of-Experts model trained on OpenAI's Harmony response format with reasoning capabilities
- providers: [wandb]
- roles: [chat, edit]
- thinking: true
-
-- name: OpenPipe Qwen3 14B Instruct
- model: OpenPipe/Qwen3-14B-Instruct
- description: Multilingual instruction-tuned model optimized for agents
- providers: [wandb]
- roles: [chat, edit]
- thinking: false
-
-- name: Qwen3.5 35B A3B
- model: Qwen/Qwen3.5-35B-A3B
- description: Multimodal Mixture-of-Experts model for chat, reasoning, and agentic tasks
- providers: [wandb]
- roles: [chat, edit]
- thinking: true
-
-- name: Qwen3 235B A22B Thinking-2507
- model: Qwen/Qwen3-235B-A22B-Thinking-2507
- description: Mixture-of-Experts model with reasoning capabilities for structured reasoning, math, and long-form generation
- providers: [wandb]
- roles: [chat, edit]
- thinking: true
-
-- name: Qwen3 235B A22B Instruct-2507
- model: Qwen/Qwen3-235B-A22B-Instruct-2507
- description: Multilingual Mixture-of-Experts instruction model optimized for logical reasoning
- providers: [wandb]
- roles: [chat, edit]
- thinking: false
-
-- name: Qwen3 30B A3B Instruct-2507
- model: Qwen/Qwen3-30B-A3B-Instruct-2507
- description: Mixture-of-Experts model with enhanced reasoning, coding, and long-context understanding
- providers: [wandb]
- roles: [chat, edit]
- thinking: false
-
-- name: Qwen3 Coder 480B A35B
- model: Qwen/Qwen3-Coder-480B-A35B-Instruct
- description: Mixture-of-Experts model optimized for agentic coding tasks such as function calling and tool use
- providers: [wandb]
- roles: [chat, edit]
- thinking: false
-
-- name: Z.AI GLM 5.1
- model: zai-org/GLM-5.1
- description: Mixture-of-Experts model for long-horizon agentic engineering and advanced reasoning
- providers: [wandb]
- roles: [chat, edit]
- thinking: true
-
-# OpenCode Go Models
-
-- name: Kimi K2.6 (OpenCode Go)
- model: kimi-k2.6
- description: Latest Kimi Mixture-of-Experts model optimized for complex tool use and reasoning
- providers: [opencode-go]
- roles: [chat, edit]
- thinking: true
-
-- name: Kimi K2.5 (OpenCode Go)
- model: kimi-k2.5
- description: Mixture-of-Experts model optimized for complex tool use and reasoning
- providers: [opencode-go]
- roles: [chat, edit]
- thinking: true
-
-- name: MiMo V2.5 Pro (OpenCode Go)
- model: mimo-v2.5-pro
- description: MiMo V2.5 Pro coding model with enhanced reasoning
- providers: [opencode-go]
- roles: [chat, edit]
- thinking: true
-
-- name: MiMo V2.5 (OpenCode Go)
- model: mimo-v2.5
- description: MiMo V2.5 efficient coding model
- providers: [opencode-go]
- roles: [chat, edit]
- thinking: false
-
-- name: MiMo V2 Pro (OpenCode Go)
- model: mimo-v2-pro
- description: MiMo V2 Pro model for coding tasks
- providers: [opencode-go]
- roles: [chat, edit]
- thinking: false
-
-- name: MiMo V2 Omni (OpenCode Go)
- model: mimo-v2-omni
- description: MiMo V2 Omni multimodal model
- providers: [opencode-go]
- roles: [chat, edit]
- thinking: false
-
-- name: GLM 5.1 (OpenCode Go)
- model: glm-5.1
- description: Mixture-of-Experts model with user-controllable thinking/non-thinking modes
- providers: [opencode-go]
- roles: [chat, edit]
- thinking: true
-
-- name: GLM 5 (OpenCode Go)
- model: glm-5
- description: Efficient Mixture-of-Experts model for coding tasks
- providers: [opencode-go]
- roles: [chat, edit]
- thinking: false
-
-- name: MiniMax M2.7 (OpenCode Go)
- model: minimax-m2.7
- description: MiniMax M2.7 model with advanced reasoning
- providers: [opencode-go]
- roles: [chat, edit]
- thinking: true
-
-- name: MiniMax M2.5 (OpenCode Go)
- model: minimax-m2.5
- description: MiniMax M2.5 model with efficient inference
- providers: [opencode-go]
- roles: [chat, edit]
- thinking: false
-
-- name: Qwen 3.6 Plus (OpenCode Go)
- model: qwen3.6-plus
- description: Qwen 3.6 Plus for complex reasoning and coding
- providers: [opencode-go]
- roles: [chat, edit]
- thinking: true
-
-- name: Qwen 3.5 Plus (OpenCode Go)
- model: qwen3.5-plus
- description: Qwen 3.5 Plus efficient coding model
- providers: [opencode-go]
- roles: [chat, edit]
- thinking: false
-
-- name: DeepSeek V4 Pro (OpenCode Go)
- model: deepseek-v4-pro
- description: DeepSeek V4 Pro coding model with strong reasoning
- providers: [opencode-go]
- roles: [chat, edit]
- thinking: true
-
-- name: DeepSeek V4 Flash (OpenCode Go)
- model: deepseek-v4-flash
- description: DeepSeek V4 Flash low-latency coding model
- providers: [opencode-go]
- roles: [chat, edit]
- thinking: false
+anthropic:
+ - name: Claude Opus 4.7
+ model: claude-opus-4-7
+ description: "Next generation of Anthropic's Opus family, built for long-running, asynchronous agents"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image, pdf]
+ output_types: [text]
+ release_date: 2026-04-16
+ cost: {input: 5, output: 25}
+
+ - name: Claude Sonnet 4.6
+ model: claude-sonnet-4-6
+ description: "Anthropic's most capable Sonnet-class model yet, with frontier performance across coding, agents, and professional work"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image, pdf]
+ output_types: [text]
+ release_date: 2026-02-17
+ cost: {input: 3, output: 15}
+
+ - name: Claude Opus 4.6
+ model: claude-opus-4-6
+ description: "Anthropic's strongest model for coding and long-running professional tasks"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image, pdf]
+ output_types: [text]
+ release_date: 2026-02-05
+ cost: {input: 5, output: 25}
+
+ - name: Claude Opus 4.5
+ model: claude-opus-4-5
+ description: "Anthropic's frontier reasoning model optimized for complex software engineering, agentic workflows, and long-horizon computer use"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image, pdf]
+ output_types: [text]
+ release_date: 2025-11-24
+ cost: {input: 5, output: 25}
+
+ - name: Claude Haiku 4.5
+ model: claude-haiku-4-5
+ description: "Anthropic's fastest and most efficient model, delivering near-frontier intelligence at a fraction of the cost and latency of larger Claude models"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image, pdf]
+ output_types: [text]
+ release_date: 2025-10-15
+ cost: {input: 1, output: 5}
+
+ - name: Claude Sonnet 4.5
+ model: claude-sonnet-4-5
+ description: "Anthropic's most advanced Sonnet model to date, optimized for real-world agents and coding workflows"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image, pdf]
+ output_types: [text]
+ release_date: 2025-09-29
+ cost: {input: 3, output: 15}
+
+openai:
+ - name: GPT-5.5
+ model: gpt-5.5
+ description: "OpenAI's frontier model designed for complex professional workloads, building on GPT-5.4 with stronger reasoning, higher reliability, and improved token efficiency on hard tasks"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image, pdf]
+ output_types: [text]
+ release_date: 2026-04-23
+ cost: {input: 5, output: 30}
+
+ - name: GPT-5.4 mini
+ model: gpt-5.4-mini
+ description: "GPT-5.4 mini brings the core capabilities of GPT-5.4 to a faster, more efficient model optimized for high-throughput workloads"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image]
+ output_types: [text]
+ release_date: 2026-03-17
+ cost: {input: 0.75, output: 4.5}
+
+ - name: GPT-5.4 nano
+ model: gpt-5.4-nano
+ description: "Most lightweight and cost-efficient variant of the GPT-5.4 family, optimized for speed-critical and high-volume tasks"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image]
+ output_types: [text]
+ release_date: 2026-03-17
+ cost: {input: 0.2, output: 1.25}
+
+ - name: GPT-5.4
+ model: gpt-5.4
+ description: "OpenAI's latest frontier model, unifying the Codex and GPT lines into a single system"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image, pdf]
+ output_types: [text]
+ release_date: 2026-03-05
+ cost: {input: 2.5, output: 15}
+
+ - name: GPT-5.3 Chat
+ model: gpt-5.3-chat-latest
+ description: "Update to ChatGPT's most-used model that makes everyday conversations smoother, more useful, and more directly helpful"
+ roles: [chat, edit]
+ capabilities: [tool_calling]
+ input_types: [text, image]
+ output_types: [text]
+ release_date: 2026-03-03
+ cost: {input: 1.75, output: 14}
+
+ - name: GPT-5.3 Codex
+ model: gpt-5.3-codex
+ description: "OpenAI's most advanced agentic coding model, combining the frontier software engineering performance of GPT-5.2-Codex with the broader reasoning and professional knowledge capabilities of GPT-5.2"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image, pdf]
+ output_types: [text]
+ release_date: 2026-02-05
+ cost: {input: 1.75, output: 14}
+
+ - name: ChatGPT Image
+ model: chatgpt-image-latest
+ description: "Previous-generation image model used in ChatGPT for image generation and editing from text and image inputs"
+ roles: [chat, edit]
+ capabilities: []
+ input_types: [text, image]
+ output_types: [text, image]
+ release_date: 2025-12-16
+
+google:
+ - name: Gemini 3.5 Flash
+ model: gemini-3.5-flash
+ description: "Google's high-efficiency multimodal model, bringing near-Pro level coding and reasoning at Flash-tier cost and speed"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image, pdf]
+ output_types: [text]
+ release_date: 2026-05-19
+ cost: {input: 1.5, output: 9}
+
+ - name: Gemini 3.1 Flash Lite
+ model: gemini-3.1-flash-lite
+ description: "Google's GA high-efficiency multimodal model optimized for low-latency, high-volume workloads"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image, pdf]
+ output_types: [text]
+ release_date: 2026-05-07
+ cost: {input: 0.25, output: 1.5}
+
+ - name: Gemma 4 26B A4B IT
+ model: gemma-4-26b-a4b-it
+ description: "Instruction-tuned Mixture-of-Experts (MoE) model from Google DeepMind"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image]
+ output_types: [text]
+ release_date: 2026-04-02
+
+ - name: Gemma 4 31B IT
+ model: gemma-4-31b-it
+ description: "Google DeepMind's 30.7B dense multimodal model supporting text and image input with text output"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image]
+ output_types: [text]
+ release_date: 2026-04-02
+
+ - name: Nano Banana 2
+ model: gemini-3.1-flash-image-preview
+ description: "Google's image generation and editing model, delivering Pro-level visual quality at Flash speed"
+ roles: [chat, edit]
+ capabilities: [thinking]
+ input_types: [text, image, pdf]
+ output_types: [text, image]
+ release_date: 2026-02-26
+ cost: {input: 0.5, output: 60}
+
+ - name: Gemini 3.1 Pro Preview
+ model: gemini-3.1-pro-preview
+ description: "Google's frontier reasoning model, delivering enhanced software engineering performance, improved agentic reliability, and more efficient token usage across complex workflows"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image, pdf]
+ output_types: [text]
+ release_date: 2026-02-19
+ cost: {input: 2, output: 12}
+
+ - name: Gemini 3.1 Pro Preview Custom Tools
+ model: gemini-3.1-pro-preview-customtools
+ description: "Variant of Gemini 3.1 Pro with refined tool selection that avoids overusing a generic bash tool"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image, pdf]
+ output_types: [text]
+ release_date: 2026-02-19
+ cost: {input: 2, output: 12}
+
+bedrock:
+ - name: Claude Opus 4.7
+ model: anthropic.claude-opus-4-7
+ description: "Next generation of Anthropic's Opus family, built for long-running, asynchronous agents"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image, pdf]
+ output_types: [text]
+ release_date: 2026-04-16
+ cost: {input: 5, output: 25}
+
+ - name: Claude Opus 4.7 (Global)
+ model: global.anthropic.claude-opus-4-7
+ description: "Next generation of Anthropic's Opus family, built for long-running, asynchronous agents"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image, pdf]
+ output_types: [text]
+ release_date: 2026-04-16
+ cost: {input: 5, output: 25}
+
+ - name: MiniMax M2.5
+ model: minimax.minimax-m2.5
+ description: "SOTA large language model designed for real-world productivity"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text]
+ output_types: [text]
+ release_date: 2026-03-18
+ cost: {input: 0.3, output: 1.2}
+
+ - name: GLM-5
+ model: zai.glm-5
+ description: "Z.ai's flagship open-source foundation model engineered for complex systems design and long-horizon agent workflows"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text]
+ output_types: [text]
+ release_date: 2026-03-18
+ cost: {input: 1, output: 3.2}
+
+ - name: NVIDIA Nemotron 3 Super 120B A12B
+ model: nvidia.nemotron-super-3-120b
+ description: "120B-parameter open hybrid MoE model, activating just 12B parameters for maximum compute efficiency and accuracy in complex multi-agent applications"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text]
+ output_types: [text]
+ release_date: 2026-03-11
+ cost: {input: 0.15, output: 0.65}
+
+ - name: Claude Sonnet 4.6
+ model: anthropic.claude-sonnet-4-6
+ description: "Anthropic's most capable Sonnet-class model yet, with frontier performance across coding, agents, and professional work"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image, pdf]
+ output_types: [text]
+ release_date: 2026-02-17
+ cost: {input: 3, output: 15}
+
+azure:
+ - name: GPT-5.5
+ model: gpt-5.5
+ description: "OpenAI's frontier model designed for complex professional workloads, building on GPT-5.4 with stronger reasoning, higher reliability, and improved token efficiency on hard tasks"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image, pdf]
+ output_types: [text]
+ release_date: 2026-04-24
+ cost: {input: 5, output: 30}
+
+ - name: Kimi K2.6
+ model: kimi-k2.6
+ description: "Moonshot AI's next-generation multimodal model, designed for long-horizon coding, coding-driven UI/UX generation, and multi-agent orchestration"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image]
+ output_types: [text]
+ release_date: 2026-04-22
+ cost: {input: 0.95, output: 4}
+
+ - name: Grok 4.20 (Non-Reasoning)
+ model: grok-4-20-non-reasoning
+ description: "Reasoning model from xAI with industry-leading speed and agentic tool calling capabilities"
+ roles: [chat, edit]
+ capabilities: [tool_calling]
+ input_types: [text]
+ output_types: [text]
+ release_date: 2026-04-08
+ cost: {input: 2, output: 6}
+
+ - name: Grok 4.20 (Reasoning)
+ model: grok-4-20-reasoning
+ description: "Reasoning model from xAI with industry-leading speed and agentic tool calling capabilities"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text]
+ output_types: [text]
+ release_date: 2026-04-08
+ cost: {input: 2, output: 6}
+
+ - name: GPT-5.4 Mini
+ model: gpt-5.4-mini
+ description: "GPT-5.4 mini brings the core capabilities of GPT-5.4 to a faster, more efficient model optimized for high-throughput workloads"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image, pdf]
+ output_types: [text]
+ release_date: 2026-03-17
+ cost: {input: 0.75, output: 4.5}
+
+ - name: GPT-5.4 Nano
+ model: gpt-5.4-nano
+ description: "Most lightweight and cost-efficient variant of the GPT-5.4 family, optimized for speed-critical and high-volume tasks"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image, pdf]
+ output_types: [text]
+ release_date: 2026-03-17
+ cost: {input: 0.2, output: 1.25}
+
+ - name: GPT-5.4
+ model: gpt-5.4
+ description: "OpenAI's latest frontier model, unifying the Codex and GPT lines into a single system"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image, pdf]
+ output_types: [text]
+ release_date: 2026-03-05
+ cost: {input: 2.5, output: 15}
+
+ - name: GPT-5.3 Chat
+ model: gpt-5.3-chat
+ description: "Update to ChatGPT's most-used model that makes everyday conversations smoother, more useful, and more directly helpful"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image]
+ output_types: [text]
+ release_date: 2026-03-03
+ cost: {input: 1.75, output: 14}
+
+ - name: GPT-5.3 Codex
+ model: gpt-5.3-codex
+ description: "OpenAI's most advanced agentic coding model, combining the frontier software engineering performance of GPT-5.2-Codex with the broader reasoning and professional knowledge capabilities of GPT-5.2"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image]
+ output_types: [text]
+ release_date: 2026-02-24
+ cost: {input: 1.75, output: 14}
+
+ - name: Claude Sonnet 4.6
+ model: claude-sonnet-4-6
+ description: ""
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image, pdf]
+ output_types: [text]
+ release_date: 2026-02-17
+ cost: {input: 3, output: 15}
+
+github:
+ - name: DeepSeek-R1-0528
+ model: deepseek/deepseek-r1-0528
+ description: "May 28th update to DeepSeek R1, with performance on par with OpenAI o1 but open-sourced and with fully open reasoning tokens"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text]
+ output_types: [text]
+ release_date: 2025-05-28
+ cost: {input: 0, output: 0}
+
+ - name: Mistral Medium 3 (25.05)
+ model: mistral-ai/mistral-medium-2505
+ description: "High-performance enterprise-grade language model designed to deliver frontier-level capabilities at significantly reduced operational cost"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image]
+ output_types: [text]
+ release_date: 2025-05-01
+ cost: {input: 0, output: 0}
+
+ - name: GPT-4.1
+ model: openai/gpt-4.1
+ description: "Flagship large language model optimized for advanced instruction following, real-world software engineering, and long-context reasoning"
+ roles: [chat, edit]
+ capabilities: [tool_calling]
+ input_types: [text, image]
+ output_types: [text]
+ release_date: 2025-04-14
+ cost: {input: 0, output: 0}
+
+ - name: GPT-4.1-mini
+ model: openai/gpt-4.1-mini
+ description: "Mid-sized model delivering performance competitive with GPT-4o at substantially lower latency and cost"
+ roles: [chat, edit]
+ capabilities: [tool_calling]
+ input_types: [text, image]
+ output_types: [text]
+ release_date: 2025-04-14
+ cost: {input: 0, output: 0}
+
+ - name: GPT-4.1-nano
+ model: openai/gpt-4.1-nano
+ description: "For tasks that demand low latency, GPT‑4.1 nano is the fastest and cheapest model in the GPT-4.1 series"
+ roles: [chat, edit]
+ capabilities: [tool_calling]
+ input_types: [text, image]
+ output_types: [text]
+ release_date: 2025-04-14
+ cost: {input: 0, output: 0}
+
+ - name: DeepSeek-V3-0324
+ model: deepseek/deepseek-v3-0324
+ description: "Latest model from the DeepSeek team, building upon the instruction following and coding abilities of the previous versions"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text]
+ output_types: [text]
+ release_date: 2025-03-24
+ cost: {input: 0, output: 0}
+
+ - name: Mistral Small 3.1
+ model: mistral-ai/mistral-small-2503
+ description: "Upgraded variant of Mistral Small 3 (2501), featuring 24 billion parameters with advanced multimodal capabilities"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image]
+ output_types: [text]
+ release_date: 2025-03-01
+ cost: {input: 0, output: 0}
+
+ - name: Llama 4 Maverick 17B 128E Instruct FP8
+ model: meta/llama-4-maverick-17b-128e-instruct-fp8
+ description: "High-capacity multimodal language model from Meta, built on a mixture-of-experts (MoE) architecture with 128 experts and 17 billion active parameters per forward"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image]
+ output_types: [text]
+ release_date: 2025-01-31
+ cost: {input: 0, output: 0}
+
+ - name: Llama 4 Scout 17B 16E Instruct
+ model: meta/llama-4-scout-17b-16e-instruct
+ description: "Mixture-of-experts (MoE) language model developed by Meta, activating 17 billion parameters out of a total of 109B"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image]
+ output_types: [text]
+ release_date: 2025-01-31
+ cost: {input: 0, output: 0}
+
+ - name: OpenAI o3
+ model: openai/o3
+ description: "Well-rounded reasoning model setting a new standard for math, science, coding, and visual reasoning tasks"
+ roles: [chat, edit]
+ capabilities: [thinking]
+ input_types: [text, image]
+ output_types: [text]
+ release_date: 2025-01-31
+ cost: {input: 0, output: 0}
+
+ollama:
+ - name: DeepSeek V4 Flash
+ model: deepseek-v4-flash
+ description: "Efficiency-optimized Mixture-of-Experts model from DeepSeek with 284B total parameters and 13B activated parameters, supporting a 1M-token context window"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text]
+ output_types: [text]
+ release_date: 2026-04-24
+
+ - name: DeepSeek V4 Pro
+ model: deepseek-v4-pro
+ description: "Large-scale Mixture-of-Experts model from DeepSeek with 1.6T total parameters and 49B activated parameters, supporting a 1M-token context window"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text]
+ output_types: [text]
+ release_date: 2026-04-24
+
+ - name: Kimi K2.6
+ model: kimi-k2.6
+ description: "Moonshot AI's next-generation multimodal model, designed for long-horizon coding, coding-driven UI/UX generation, and multi-agent orchestration"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image]
+ output_types: [text]
+ release_date: 2026-04-20
+
+ - name: Gemma 4 31B
+ model: gemma4:31b
+ description: "Google DeepMind's 30.7B dense multimodal model supporting text and image input with text output"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image]
+ output_types: [text]
+ release_date: 2026-04-02
+
+ - name: GLM 5.1
+ model: glm-5.1
+ description: "GLM-5.1 delivers a major leap in coding capability, with particularly significant gains in handling long-horizon tasks"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text]
+ output_types: [text]
+ release_date: 2026-03-27
+
+ - name: MiniMax M2.7
+ model: minimax-m2.7
+ description: "Next-generation large language model designed for autonomous, real-world productivity and continuous improvement"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text]
+ output_types: [text]
+ release_date: 2026-03-18
+
+ - name: NVIDIA Nemotron 3 Super 120B
+ model: nemotron-3-super
+ description: "120B-parameter open hybrid MoE model, activating just 12B parameters for maximum compute efficiency and accuracy in complex multi-agent applications"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text]
+ output_types: [text]
+ release_date: 2026-03-11
+
+ - name: Qwen 3.5 397B
+ model: qwen3.5:397b
+ description: "Built on a hybrid architecture that integrates a linear attention mechanism with a sparse mixture-of-experts model, achieving higher inference efficiency"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image]
+ output_types: [text]
+ release_date: 2026-02-15
+
+ - name: MiniMax M2.5
+ model: minimax-m2.5
+ description: "SOTA large language model designed for real-world productivity"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text]
+ output_types: [text]
+ release_date: 2026-02-12
+
+ - name: GLM 5
+ model: glm-5
+ description: "Z.ai's flagship open-source foundation model engineered for complex systems design and long-horizon agent workflows"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text]
+ output_types: [text]
+ release_date: 2026-02-11
+
+wandb:
+ - name: GLM-5.1
+ model: zai-org/GLM-5.1
+ description: "GLM-5.1 delivers a major leap in coding capability, with particularly significant gains in handling long-horizon tasks"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text]
+ output_types: [text]
+ release_date: 2026-03-27
+ cost: {input: 1.4, output: 4.4}
+
+ - name: NVIDIA Nemotron 3 Super 120B
+ model: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8
+ description: "120B-parameter open hybrid MoE model, activating just 12B parameters for maximum compute efficiency and accuracy in complex multi-agent applications"
+ roles: [chat, edit]
+ capabilities: [tool_calling]
+ input_types: [text]
+ output_types: [text]
+ release_date: 2026-03-11
+ cost: {input: 0.2, output: 0.8}
+
+ - name: MiniMax M2.5
+ model: MiniMaxAI/MiniMax-M2.5
+ description: "SOTA large language model designed for real-world productivity"
+ roles: [chat, edit]
+ capabilities: [tool_calling]
+ input_types: [text]
+ output_types: [text]
+ release_date: 2026-02-12
+ cost: {input: 0.3, output: 1.2}
+
+ - name: GLM 5
+ model: zai-org/GLM-5-FP8
+ description: "Z.ai's flagship open-source foundation model engineered for complex systems design and long-horizon agent workflows"
+ roles: [chat, edit]
+ capabilities: [tool_calling]
+ input_types: [text]
+ output_types: [text]
+ release_date: 2026-02-11
+ cost: {input: 1, output: 3.2}
+
+ - name: Kimi K2.5
+ model: moonshotai/Kimi-K2.5
+ description: "Moonshot AI's native multimodal model, delivering state-of-the-art visual coding capability and a self-directed agent swarm paradigm"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image]
+ output_types: [text]
+ release_date: 2026-01-27
+ cost: {input: 0.5, output: 2.85}
+
+ - name: DeepSeek V3.1
+ model: deepseek-ai/DeepSeek-V3.1
+ description: "Large hybrid reasoning model (671B parameters, 37B active) that supports both thinking and non-thinking modes via prompt templates"
+ roles: [chat, edit]
+ capabilities: [tool_calling]
+ input_types: [text]
+ output_types: [text]
+ release_date: 2025-08-21
+ cost: {input: 0.55, output: 1.65}
+
+ - name: gpt-oss-120b
+ model: openai/gpt-oss-120b
+ description: "Open-weight, 117B-parameter Mixture-of-Experts (MoE) language model from OpenAI designed for high-reasoning, agentic, and general-purpose production use cases"
+ roles: [chat, edit]
+ capabilities: [tool_calling]
+ input_types: [text]
+ output_types: [text]
+ release_date: 2025-08-05
+ cost: {input: 0.15, output: 0.6}
+
+ - name: gpt-oss-20b
+ model: openai/gpt-oss-20b
+ description: "Open-weight 21B parameter model released by OpenAI under the Apache 2.0 license"
+ roles: [chat, edit]
+ capabilities: [tool_calling]
+ input_types: [text]
+ output_types: [text]
+ release_date: 2025-08-05
+ cost: {input: 0.05, output: 0.2}
+
+ - name: Qwen3 30B A3B Instruct 2507
+ model: Qwen/Qwen3-30B-A3B-Instruct-2507
+ description: "30.5B-parameter mixture-of-experts language model from Qwen, with 3.3B active parameters per inference"
+ roles: [chat, edit]
+ capabilities: [tool_calling]
+ input_types: [text]
+ output_types: [text]
+ release_date: 2025-07-29
+ cost: {input: 0.1, output: 0.3}
+
+ - name: Qwen3-235B-A22B-Thinking-2507
+ model: Qwen/Qwen3-235B-A22B-Thinking-2507
+ description: "High-performance, open-weight Mixture-of-Experts (MoE) language model optimized for complex reasoning tasks"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text]
+ output_types: [text]
+ release_date: 2025-07-25
+ cost: {input: 0.1, output: 0.1}
+
+opencode-go:
+ - name: DeepSeek V4 Flash
+ model: deepseek-v4-flash
+ description: "Efficiency-optimized Mixture-of-Experts model from DeepSeek with 284B total parameters and 13B activated parameters, supporting a 1M-token context window"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text]
+ output_types: [text]
+ release_date: 2026-04-24
+ cost: {input: 0.14, output: 0.28}
+
+ - name: DeepSeek V4 Pro
+ model: deepseek-v4-pro
+ description: "Large-scale Mixture-of-Experts model from DeepSeek with 1.6T total parameters and 49B activated parameters, supporting a 1M-token context window"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text]
+ output_types: [text]
+ release_date: 2026-04-24
+ cost: {input: 1.74, output: 3.48}
+
+ - name: MiMo V2.5
+ model: mimo-v2.5
+ description: "Native omnimodal model from Xiaomi delivering Pro-level agentic performance at roughly half the inference cost"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image]
+ output_types: [text]
+ release_date: 2026-04-22
+ cost: {input: 0.4, output: 2}
+
+ - name: MiMo V2.5 Pro
+ model: mimo-v2.5-pro
+ description: "Xiaomi's flagship model, delivering strong performance in general agentic capabilities, complex software engineering, and long-horizon tasks, with top rankings on benchmarks such as ClawEval, GDPVal,…"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text]
+ output_types: [text]
+ release_date: 2026-04-22
+ cost: {input: 1, output: 3}
+
+ - name: Kimi K2.6
+ model: kimi-k2.6
+ description: "Moonshot AI's next-generation multimodal model, designed for long-horizon coding, coding-driven UI/UX generation, and multi-agent orchestration"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image]
+ output_types: [text]
+ release_date: 2026-04-21
+ cost: {input: 0.95, output: 4}
+
+ - name: GLM-5.1
+ model: glm-5.1
+ description: "GLM-5.1 delivers a major leap in coding capability, with particularly significant gains in handling long-horizon tasks"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text]
+ output_types: [text]
+ release_date: 2026-04-07
+ cost: {input: 1.4, output: 4.4}
+
+ - name: Qwen3.6 Plus
+ model: qwen3.6-plus
+ description: "Qwen 3.6 Plus builds on a hybrid architecture that combines efficient linear attention with sparse mixture-of-experts routing, enabling strong scalability and high-performance inference"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image]
+ output_types: [text]
+ release_date: 2026-04-02
+ cost: {input: 0.5, output: 3}
+
+ - name: MiMo V2 Omni
+ model: mimo-v2-omni
+ description: "Frontier omni-modal model that natively processes image, video, and audio inputs within a unified architecture"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image, pdf]
+ output_types: [text]
+ release_date: 2026-03-18
+ cost: {input: 0.4, output: 2}
+
+ - name: MiMo V2 Pro
+ model: mimo-v2-pro
+ description: "Xiaomi's flagship foundation model, featuring over 1T total parameters and a 1M context length, deeply optimized for agentic scenarios"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text]
+ output_types: [text]
+ release_date: 2026-03-18
+ cost: {input: 1, output: 3}
+
+ - name: MiniMax M2.7
+ model: minimax-m2.7
+ description: "Next-generation large language model designed for autonomous, real-world productivity and continuous improvement"
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text]
+ output_types: [text]
+ release_date: 2026-03-18
+ cost: {input: 0.3, output: 1.2}
diff --git a/packages/llm-info/package.json b/packages/llm-info/package.json
index fb138bf03c..575e523350 100644
--- a/packages/llm-info/package.json
+++ b/packages/llm-info/package.json
@@ -15,6 +15,7 @@
},
"scripts": {
"codegen": "node --experimental-transform-types src/generate.ts",
+ "sync-models": "node --experimental-transform-types src/sync-models.ts",
"test": "vitest",
"build": "echo 'No build step'"
},
diff --git a/packages/llm-info/skills/SKILL.md b/packages/llm-info/skills/SKILL.md
new file mode 100644
index 0000000000..2b9abc550a
--- /dev/null
+++ b/packages/llm-info/skills/SKILL.md
@@ -0,0 +1,98 @@
+---
+name: fill-model-descriptions
+description: Fill empty `description: ""` fields in `packages/llm-info/data/models.yml` by querying OpenRouter and provider documentation. Use when the user asks to populate model descriptions, enrich the model catalog, or backfill descriptions after running `pnpm sync-models`.
+disable-model-invocation: true
+---
+
+# Fill Model Descriptions
+
+## Goal
+
+Replace empty `description: ""` strings in `packages/llm-info/data/models.yml` with concise, accurate one-sentence summaries sourced from OpenRouter first, then provider documentation as a fallback.
+
+## Scope and Contract
+
+- **Only modify entries where `description` is the empty string.** Existing non-empty descriptions are human-curated; never overwrite them.
+- The file is a top-level YAML map keyed by provider id (`anthropic:`, `openai:`, …). Each provider's value is an array of model entries.
+- Do not modify any other field, reorder entries, or change formatting (flow-style arrays like `roles: [chat, edit]`, blank lines between entries, blank line between provider sections).
+- This skill requires network access.
+
+## Workflow
+
+1. **Read** `packages/llm-info/data/models.yml`. Parse it with the `yaml` library's `parseDocument` (NOT `parse`) so formatting and comments are preserved on write-back.
+
+2. **Collect** every entry where `description` is the empty string. Track them as `{ provider, model }` pairs.
+
+3. **Source descriptions** in this priority order:
+
+ **a. OpenRouter** — `GET https://openrouter.ai/api/v1/models` (no auth required). Build a lookup keyed by lowercased `id`. For each `{ provider, model }`, try candidates in this order:
+ - `${vendor}/${model}` where `vendor` comes from the Vendor Map below.
+ - `${vendor}/${model.replaceAll('-', '.')}` (some OR ids use `.` separators, e.g. `claude-3.5-sonnet`).
+ - For Bedrock entries with region/alias prefixes (`us.`, `eu.`, `jp.`, `global.`, `anthropic.`, `minimax.`, `mistral.`, `zai.`, `nvidia.`), strip the prefix and retry with the appropriate vendor.
+ - For Google entries with `@default` suffix (e.g. `claude-opus-4-7@default`), strip the suffix and try `anthropic/${rest}`.
+
+ **b. Provider docs** — if no OpenRouter hit, search the canonical provider's docs page (anthropic.com/news, platform.openai.com/docs/models, ai.google.dev/gemini-api/docs/models, mistral.ai/news, x.ai/blog, etc.) for the model's one-line summary. Use the `WebSearch` or `WebFetch` tool.
+
+ **c. Skip** — if neither source has a confident answer, leave `description: ""` and record the `{ provider, model }` for the final summary. Empty is better than wrong.
+
+4. **Truncate** each sourced description per the rules in the "Truncation" section.
+
+5. **Write** the YAML back using targeted `Pair` mutations on the parsed `Document`, not text replacement. Reference implementation: `packages/llm-info/src/sync-models.ts` shows how to load a Document, mutate sequence/map nodes, and stringify with `{ lineWidth: 0, flowCollectionPadding: false }` to preserve formatting.
+
+6. **Validate** by running:
+ ```bash
+ pnpm --filter @marimo-team/llm-info test
+ ```
+ The `schema.test.ts` test will catch any structural drift.
+
+7. **Report** to the user:
+ - How many entries got descriptions.
+ - How many were skipped (with the provider/model list).
+
+## Vendor Map
+
+OpenRouter id vendor prefixes for each marimo provider:
+
+| Marimo provider | OpenRouter vendor prefix | Notes |
+|---|---|---|
+| `anthropic` | `anthropic` | Direct match in most cases |
+| `openai` | `openai` | Direct match |
+| `google` | `google` | Strip `@default` suffix if present |
+| `mistral` | `mistralai` | Note the `ai` suffix |
+| `xai` | `x-ai` | Hyphenated |
+| `azure` | `openai` | Azure mostly re-exposes OpenAI models |
+| `bedrock` | varies | Strip region prefix (`us.`/`eu.`/`jp.`/`global.`) and use the embedded vendor (`anthropic.foo` → `anthropic/foo`) |
+| `github` | varies | Ids like `openai/gpt-4.1` carry the vendor; strip the github layer |
+| `openrouter` | embedded in id | Ids like `anthropic/claude-opus-4.7-fast` already carry the vendor |
+| `wandb` | embedded in id | Ids like `deepseek-ai/DeepSeek-V3.1`; the prefix is the vendor |
+| `opencode-go` | rarely on OR | Source from provider docs |
+| `ollama` | not on OR | Source from the Ollama model card |
+
+## Truncation
+
+- **One sentence.** Split on `. ` and take the first. Strip trailing whitespace and any markdown formatting (`**`, `_`, backticks).
+- **Hard cap at 200 characters.** If the first sentence is longer, cut at the last word boundary before 200 and append `…`.
+- **Strip leading articles** for consistency: prefer "Frontier reasoning model optimized for…" over "This is a frontier reasoning model optimized for…".
+- **Prefer capability statements over marketing.** If OpenRouter gives "the best model from X" but the provider docs give "optimized for tool use and long-context reasoning", prefer the latter.
+
+## Examples
+
+**Good** (matches the tone of existing hand-curated entries):
+- `Latest Opus model, strongest for coding and long-running professional tasks`
+- `Most capable Sonnet-class model, with frontier performance across coding, agents, and professional work`
+- `Lightweight Gemma model trained by Google, designed to run on a single GPU`
+- `Multimodal Mixture-of-Experts model optimized for complex tool use and reasoning`
+
+**Bad** (do not produce these):
+- `Claude 3.5 Sonnet delivers better-than-Opus capabilities, faster-than-Sonnet speeds, at the same Sonnet prices. Sonnet is particularly good at coding, vision…` — too long, contains marketing
+- `The best model from Anthropic` — vague, no information content
+- `An AI model` — useless
+- `**Claude 3.5 Sonnet** is a _frontier_ model…` — markdown not stripped
+
+## Quick Reference
+
+- Data file: `packages/llm-info/data/models.yml`
+- Schema: `packages/llm-info/src/index.ts` (`AiModel` interface)
+- YAML write reference: `packages/llm-info/src/sync-models.ts` (`buildEntryNode`, `addProviderSection`)
+- OpenRouter API: `https://openrouter.ai/api/v1/models` (public, no auth)
+- Validation: `pnpm --filter @marimo-team/llm-info test`
diff --git a/packages/llm-info/src/__tests__/cli.test.ts b/packages/llm-info/src/__tests__/cli.test.ts
new file mode 100644
index 0000000000..eb88d39719
--- /dev/null
+++ b/packages/llm-info/src/__tests__/cli.test.ts
@@ -0,0 +1,101 @@
+/* Copyright 2026 Marimo. All rights reserved. */
+
+import { describe, expect, it, vi } from "vitest";
+import { parseCliArgs } from "../cli.ts";
+
+describe("parseCliArgs / mode", () => {
+ it("defaults to append", () => {
+ expect(parseCliArgs([]).mode).toBe("append");
+ });
+
+ it.each([
+ ["--replace"],
+ ["-r"],
+ ["--mode=replace"],
+ // Regression: the two-token form used to silently fall through to append.
+ ["--mode", "replace"],
+ ])("recognises %j as replace mode", (...flags: string[]) => {
+ expect(parseCliArgs(flags).mode).toBe("replace");
+ });
+
+ it("accepts --mode=append explicitly", () => {
+ expect(parseCliArgs(["--mode=append"]).mode).toBe("append");
+ expect(parseCliArgs(["--mode", "append"]).mode).toBe("append");
+ });
+
+ it("rejects unknown --mode values", () => {
+ expect(() => parseCliArgs(["--mode", "wipe"])).toThrow(
+ /Unknown --mode value/,
+ );
+ });
+});
+
+describe("parseCliArgs / max-per-provider", () => {
+ it("returns undefined when not provided", () => {
+ expect(parseCliArgs([]).maxPerProvider).toBeUndefined();
+ });
+
+ it.each([
+ ["--max-per-provider=5", 5],
+ ["--max=3", 3],
+ ["-n=7", 7],
+ ])("parses %s as %i", (flag, expected) => {
+ expect(parseCliArgs([flag]).maxPerProvider).toBe(expected);
+ });
+
+ it("parses the two-token form `-n 4`", () => {
+ expect(parseCliArgs(["-n", "4"]).maxPerProvider).toBe(4);
+ });
+
+ it("ignores non-positive / non-numeric values", () => {
+ expect(parseCliArgs(["-n", "0"]).maxPerProvider).toBeUndefined();
+ expect(parseCliArgs(["-n", "abc"]).maxPerProvider).toBeUndefined();
+ });
+});
+
+describe("parseCliArgs / providers", () => {
+ it("returns undefined when not provided", () => {
+ expect(parseCliArgs([]).providers).toBeUndefined();
+ });
+
+ it("parses a single known provider", () => {
+ expect(parseCliArgs(["--provider=anthropic"]).providers).toEqual([
+ "anthropic",
+ ]);
+ });
+
+ it("parses a comma-separated list and trims whitespace", () => {
+ expect(parseCliArgs(["-p", "anthropic, openai ,google"]).providers).toEqual(
+ ["anthropic", "openai", "google"],
+ );
+ });
+
+ it("warns about unknown providers but keeps the known ones", () => {
+ const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
+ try {
+ expect(parseCliArgs(["-p", "anthropic,bogus"]).providers).toEqual([
+ "anthropic",
+ ]);
+ expect(warn).toHaveBeenCalled();
+ } finally {
+ warn.mockRestore();
+ }
+ });
+
+ // Regression: previously returned `[]`, which under --replace silently
+ // wiped models.yml. Must throw so the script exits non-zero instead.
+ it("throws when every requested provider is unknown", () => {
+ const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
+ try {
+ expect(() => parseCliArgs(["-p", "bogus,typo"])).toThrow(
+ /No known providers/,
+ );
+ } finally {
+ warn.mockRestore();
+ }
+ });
+
+ it("throws on an empty --provider value", () => {
+ expect(() => parseCliArgs(["--provider="])).toThrow(/no value parsed/);
+ });
+});
diff --git a/packages/llm-info/src/__tests__/json-structure.test.ts b/packages/llm-info/src/__tests__/json-structure.test.ts
index 522c9e49ff..166228d732 100644
--- a/packages/llm-info/src/__tests__/json-structure.test.ts
+++ b/packages/llm-info/src/__tests__/json-structure.test.ts
@@ -1,12 +1,18 @@
import { describe, expect, it } from "vitest";
describe("JSON structure should be correct", async () => {
- it("should have the correct structure, models should be an object", async () => {
+ it("models.json wraps a provider-keyed map under `models`", async () => {
const models = await import("../../data/generated/models.json");
- expect(Array.isArray(models.models)).toBe(true);
+ expect(typeof models.models).toBe("object");
+ expect(Array.isArray(models.models)).toBe(false);
+ for (const [, list] of Object.entries(
+ models.models as Record<string, unknown>,
+ )) {
+ expect(Array.isArray(list)).toBe(true);
+ }
});
- it("should have the correct structure, providers should be an object", async () => {
+ it("providers.json wraps an array under `providers`", async () => {
const providers = await import("../../data/generated/providers.json");
expect(Array.isArray(providers.providers)).toBe(true);
});
diff --git a/packages/llm-info/src/__tests__/schema.test.ts b/packages/llm-info/src/__tests__/schema.test.ts
index 24fc82d8b4..45792d1c05 100644
--- a/packages/llm-info/src/__tests__/schema.test.ts
+++ b/packages/llm-info/src/__tests__/schema.test.ts
@@ -2,62 +2,62 @@ import { readFileSync } from "node:fs";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import { parse } from "yaml";
-import { LLMInfoSchema, ProviderSchema } from "../generate";
+import {
+ LLMInfoSchema,
+ ModelsByProviderSchema,
+ ProviderSchema,
+} from "../generate";
describe("LLMInfoSchema", () => {
- it("should validate a valid LLM info object", () => {
+ it("validates a complete model entry", () => {
const validModel = {
name: "Test Model",
model: "test-model-v1",
description: "A test model for validation",
- providers: ["test-provider"],
roles: ["chat", "edit"],
- thinking: true,
+ capabilities: ["thinking", "tool_calling"],
+ input_types: ["text", "image"],
+ output_types: ["text"],
+ release_date: "2026-01-15",
};
- expect(() => LLMInfoSchema.parse(validModel)).not.toThrow();
- const result = LLMInfoSchema.parse(validModel);
- expect(result).toEqual(validModel);
+ expect(LLMInfoSchema.parse(validModel)).toEqual(validModel);
});
- it("should set thinking to false by default", () => {
- const modelWithoutThinking = {
+ it("defaults capabilities / input_types / output_types to empty arrays", () => {
+ const minimal = {
name: "Test Model",
model: "test-model-v1",
- description: "A test model for validation",
- providers: ["test-provider"],
+ description: "A test model",
roles: ["chat"],
};
- const result = LLMInfoSchema.parse(modelWithoutThinking);
- expect(result.thinking).toBe(false);
+ const parsed = LLMInfoSchema.parse(minimal);
+ expect(parsed.capabilities).toEqual([]);
+ expect(parsed.input_types).toEqual([]);
+ expect(parsed.output_types).toEqual([]);
});
- it("should reject invalid roles", () => {
+ it("rejects unknown roles", () => {
const invalidModel = {
name: "Test Model",
model: "test-model-v1",
- description: "A test model for validation",
- providers: ["test-provider"],
+ description: "A test model",
roles: ["invalid-role"],
- thinking: false,
};
expect(() => LLMInfoSchema.parse(invalidModel)).toThrow();
});
- it("should require all mandatory fields", () => {
- const incompleteModel = {
- name: "Test Model",
- model: "test-model-v1",
- };
-
- expect(() => LLMInfoSchema.parse(incompleteModel)).toThrow();
+ it("requires the core identity fields", () => {
+ expect(() =>
+ LLMInfoSchema.parse({ name: "Test Model", model: "test-model-v1" }),
+ ).toThrow();
});
});
describe("ProviderSchema", () => {
- it("should validate a valid provider object", () => {
+ it("validates a valid provider object", () => {
const validProvider = {
name: "Test Provider",
id: "test-provider",
@@ -65,45 +65,35 @@ describe("ProviderSchema", () => {
url: "https://example.com",
};
- expect(() => ProviderSchema.parse(validProvider)).not.toThrow();
- const result = ProviderSchema.parse(validProvider);
- expect(result).toEqual(validProvider);
+ expect(ProviderSchema.parse(validProvider)).toEqual(validProvider);
});
- it("should reject invalid URLs", () => {
- const invalidProvider = {
- name: "Test Provider",
- id: "test-provider",
- description: "A test provider for validation",
- url: "not-a-valid-url",
- };
-
- expect(() => ProviderSchema.parse(invalidProvider)).toThrow();
+ it("rejects invalid URLs", () => {
+ expect(() =>
+ ProviderSchema.parse({
+ name: "Test Provider",
+ id: "test-provider",
+ description: "A test provider",
+ url: "not-a-valid-url",
+ }),
+ ).toThrow();
});
- it("should require all fields", () => {
- const incompleteProvider = {
- name: "Test Provider",
- id: "test-provider",
- };
-
- expect(() => ProviderSchema.parse(incompleteProvider)).toThrow();
+ it("requires all fields", () => {
+ expect(() =>
+ ProviderSchema.parse({ name: "Test Provider", id: "test-provider" }),
+ ).toThrow();
});
});
describe("Data Validation", () => {
- it("should validate all models in data/models.yml", () => {
+ it("validates the structure of data/models.yml as a provider-keyed map", () => {
const modelsPath = join(__dirname, "../../data/models.yml");
const modelsYaml = readFileSync(modelsPath, "utf-8");
- const models = parse(modelsYaml);
-
- expect(Array.isArray(models)).toBe(true);
- expect(models.length).toBeGreaterThan(0);
+ const data = parse(modelsYaml);
- models.forEach((model: any, index: number) => {
- expect(() => LLMInfoSchema.parse(model)).not.toThrow(
- `Model at index ${index} failed validation: ${JSON.stringify(model)}`,
- );
- });
+ expect(typeof data).toBe("object");
+ expect(Array.isArray(data)).toBe(false);
+ expect(() => ModelsByProviderSchema.parse(data)).not.toThrow();
});
});
diff --git a/packages/llm-info/src/__tests__/sync-models.test.ts b/packages/llm-info/src/__tests__/sync-models.test.ts
new file mode 100644
index 0000000000..b9d08d1cc3
--- /dev/null
+++ b/packages/llm-info/src/__tests__/sync-models.test.ts
@@ -0,0 +1,887 @@
+/* Copyright 2026 Marimo. All rights reserved. */
+
+import { mkdtempSync, readFileSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { beforeEach, describe, expect, it } from "vitest";
+import { parse } from "yaml";
+import {
+ MAX_COST_INPUT,
+ MAX_COST_OUTPUT,
+ MAX_MODELS_PER_PROVIDER,
+ MODEL_DENYLIST,
+ mergeModels,
+} from "../sources/merge.ts";
+import type { ModelsDevApi } from "../sources/models-dev.ts";
+import { syncModels } from "../sync-models.ts";
+
+const FIXTURE_YAML = `# Manually curated section
+
+anthropic:
+ - name: Claude Opus 4.5
+ model: claude-opus-4-5
+ description: A hand-written description.
+ roles: [chat, edit]
+ capabilities: [thinking, tool_calling]
+ input_types: [text, image, pdf]
+ output_types: [text]
+ release_date: 2025-11-01
+`;
+
+const FIXTURE_API: ModelsDevApi = {
+ anthropic: {
+ id: "anthropic",
+ name: "Anthropic",
+ models: {
+ // Already in FIXTURE_YAML under `anthropic:` — should be preserved.
+ "claude-opus-4-5": {
+ id: "claude-opus-4-5",
+ name: "Claude Opus 4.5 (upstream-name-should-not-overwrite)",
+ reasoning: true,
+ tool_call: true,
+ release_date: "2025-11-01",
+ modalities: { input: ["text", "image", "pdf"], output: ["text"] },
+ },
+ "claude-opus-4-7": {
+ id: "claude-opus-4-7",
+ name: "Claude Opus 4.7",
+ reasoning: true,
+ tool_call: true,
+ release_date: "2026-04-15",
+ modalities: { input: ["text", "image"], output: ["text"] },
+ cost: { input: 15, output: 75 },
+ },
+ },
+ },
+ openai: {
+ id: "openai",
+ name: "OpenAI",
+ models: {
+ "gpt-5-mini": {
+ id: "gpt-5-mini",
+ name: "GPT-5 Mini",
+ reasoning: false,
+ tool_call: true,
+ release_date: "2025-08-01",
+ modalities: { input: ["text"], output: ["text"] },
+ },
+ // Also exposed via azure below — should appear under both providers.
+ "gpt-5.5": {
+ id: "gpt-5.5",
+ name: "GPT-5.5",
+ reasoning: true,
+ tool_call: true,
+ release_date: "2026-01-15",
+ modalities: { input: ["text", "image", "audio"], output: ["text"] },
+ },
+ // Missing `release_date` — should fall back to the epoch sentinel.
+ "text-embedding-3-large": {
+ id: "text-embedding-3-large",
+ name: "OpenAI Text Embedding 3 Large",
+ reasoning: false,
+ },
+ },
+ },
+ azure: {
+ id: "azure",
+ name: "Azure",
+ models: {
+ "gpt-5.5": {
+ id: "gpt-5.5",
+ name: "GPT-5.5",
+ reasoning: true,
+ tool_call: true,
+ release_date: "2026-01-15",
+ },
+ },
+ },
+ // Should be ignored entirely (provider not in PROVIDER_MAP).
+ "302ai": {
+ id: "302ai",
+ name: "302 AI",
+ models: {
+ "some-obscure-model": {
+ id: "some-obscure-model",
+ name: "Some Obscure Model",
+ },
+ },
+ },
+};
+
+describe("mergeModels", () => {
+ it("preserves existing entries (matched by provider + model id) and adds new ones", () => {
+ const existing = { anthropic: [{ model: "claude-opus-4-5" }] };
+ const summary = mergeModels(existing, FIXTURE_API);
+
+ expect(summary.preservedCount).toBe(1);
+ expect(summary.skippedExisting).toEqual(["anthropic/claude-opus-4-5"]);
+ expect(Object.keys(summary.newEntries).sort()).toEqual([
+ "anthropic",
+ "azure",
+ "openai",
+ ]);
+ expect(summary.newEntries["anthropic"]!.map((e) => e.model)).toEqual([
+ "claude-opus-4-7",
+ ]);
+ expect(summary.newEntries["openai"]!.map((e) => e.model)).toEqual([
+ "gpt-5.5",
+ "gpt-5-mini",
+ "text-embedding-3-large",
+ ]);
+ expect(summary.newEntries["azure"]!.map((e) => e.model)).toEqual([
+ "gpt-5.5",
+ ]);
+ });
+
+ it("emits the same model under each provider that exposes it", () => {
+ const summary = mergeModels({}, FIXTURE_API);
+ const inOpenai = summary.newEntries["openai"]!.find(
+ (e) => e.model === "gpt-5.5",
+ );
+ const inAzure = summary.newEntries["azure"]!.find(
+ (e) => e.model === "gpt-5.5",
+ );
+ expect(inOpenai).toBeDefined();
+ expect(inAzure).toBeDefined();
+ });
+
+ it("populates capabilities from upstream `reasoning`/`tool_call`", () => {
+ const summary = mergeModels({}, FIXTURE_API);
+ const opus = summary.newEntries["anthropic"]!.find(
+ (e) => e.model === "claude-opus-4-7",
+ );
+ expect(opus!.capabilities).toEqual(["thinking", "tool_calling"]);
+
+ const embed = summary.newEntries["openai"]!.find(
+ (e) => e.model === "text-embedding-3-large",
+ );
+ expect(embed!.capabilities).toEqual([]);
+ });
+
+ it("filters input/output modalities to the supported DataType enum", () => {
+ const summary = mergeModels({}, FIXTURE_API);
+ const opus = summary.newEntries["anthropic"]!.find(
+ (e) => e.model === "claude-opus-4-7",
+ );
+ expect(opus!.input_types).toEqual(["text", "image"]);
+ expect(opus!.output_types).toEqual(["text"]);
+
+ // `audio` is upstream but not in `DataType`, so it gets dropped.
+ const gpt55 = summary.newEntries["openai"]!.find(
+ (e) => e.model === "gpt-5.5",
+ );
+ expect(gpt55!.input_types).toEqual(["text", "image"]);
+ });
+
+ it("normalizes `release_date` to YYYY-MM-DD, falling back to epoch when missing", () => {
+ const summary = mergeModels({}, FIXTURE_API);
+ const opus = summary.newEntries["anthropic"]!.find(
+ (e) => e.model === "claude-opus-4-7",
+ );
+ expect(opus!.release_date).toBe("2026-04-15");
+
+ const embed = summary.newEntries["openai"]!.find(
+ (e) => e.model === "text-embedding-3-large",
+ );
+ expect(embed!.release_date).toBe("1970-01-01");
+ });
+
+ // Regression: `Date.parse` accepts more than `YYYY-MM-DD`. We must
+ // canonicalize, otherwise the YAML and the lex-sort in `sortAndTrim` break.
+ it("canonicalizes non-YYYY-MM-DD parseable inputs (timestamp, year-only)", () => {
+ const fixture: ModelsDevApi = {
+ openai: {
+ id: "openai",
+ name: "OpenAI",
+ models: {
+ // Bare year — would have left `"2026"` in the YAML otherwise.
+ a: {
+ id: "a",
+ name: "A",
+ reasoning: false,
+ tool_call: false,
+ release_date: "2026",
+ },
+ // Full ISO timestamp — would have leaked the `T…Z` suffix.
+ b: {
+ id: "b",
+ name: "B",
+ reasoning: false,
+ tool_call: false,
+ release_date: "2026-05-07T12:34:56Z",
+ },
+ // Garbage — falls back to the epoch sentinel.
+ c: {
+ id: "c",
+ name: "C",
+ reasoning: false,
+ tool_call: false,
+ release_date: "not-a-date",
+ },
+ },
+ },
+ };
+ const entries = mergeModels({}, fixture).newEntries["openai"]!;
+ const byId = Object.fromEntries(entries.map((e) => [e.model, e]));
+ expect(byId["a"]!.release_date).toBe("2026-01-01");
+ expect(byId["b"]!.release_date).toBe("2026-05-07");
+ expect(byId["c"]!.release_date).toBe("1970-01-01");
+
+ // And the sort is now correct — `b` (May 7) ranks above `a` (Jan 1).
+ expect(entries.map((e) => e.model)).toEqual(["b", "a", "c"]);
+ });
+
+ it("derives `roles: [embed]` for embedding models", () => {
+ const summary = mergeModels({}, FIXTURE_API);
+ const embedding = summary.newEntries["openai"]!.find(
+ (e) => e.model === "text-embedding-3-large",
+ );
+ expect(embedding!.roles).toEqual(["embed"]);
+ });
+
+ it("derives `roles: [chat, edit]` for general chat models", () => {
+ const summary = mergeModels({}, FIXTURE_API);
+ const chat = summary.newEntries["anthropic"]!.find(
+ (e) => e.model === "claude-opus-4-7",
+ );
+ expect(chat!.roles).toEqual(["chat", "edit"]);
+ });
+
+ it("skips models from providers outside the allowlist", () => {
+ const summary = mergeModels({}, FIXTURE_API);
+ expect(
+ (summary.newEntries as Record<string, unknown>)["302ai"],
+ ).toBeUndefined();
+ });
+
+ it("sorts new entries newest-first within each provider", () => {
+ const summary = mergeModels({}, FIXTURE_API);
+ const openaiOrder = summary.newEntries["openai"]!.map((e) => e.model);
+ expect(openaiOrder).toEqual([
+ "gpt-5.5", // 2026-01-15
+ "gpt-5-mini", // 2025-08-01
+ "text-embedding-3-large", // epoch sentinel
+ ]);
+ });
+
+ it("caps new entries to the N latest per provider", () => {
+ const make = (id: string, date: string) => ({
+ id,
+ name: id,
+ reasoning: false,
+ tool_call: false,
+ release_date: date,
+ });
+ const noisy: ModelsDevApi = {
+ anthropic: {
+ id: "anthropic",
+ models: {
+ a1: make("a1", "2020-01-01"),
+ a2: make("a2", "2021-01-01"),
+ a3: make("a3", "2022-01-01"),
+ a4: make("a4", "2023-01-01"),
+ a5: make("a5", "2024-01-01"),
+ },
+ },
+ };
+
+ const summary = mergeModels({}, noisy, { maxPerProvider: 2 });
+ expect(summary.newEntries["anthropic"]!.map((e) => e.model)).toEqual([
+ "a5",
+ "a4",
+ ]);
+ });
+
+ it("defaults to 10 per provider when no cap is passed", () => {
+ expect(MAX_MODELS_PER_PROVIDER).toBe(10);
+ });
+
+ it("dedupes models that appear under multiple mapped providers (google + google-vertex → google)", () => {
+ const fixture: ModelsDevApi = {
+ google: {
+ id: "google",
+ name: "Google",
+ models: {
+ "gemini-3.1-flash-lite": {
+ id: "gemini-3.1-flash-lite",
+ name: "Gemini 3.1 Flash Lite",
+ reasoning: true,
+ tool_call: true,
+ release_date: "2026-05-07",
+ },
+ },
+ },
+ "google-vertex": {
+ id: "google-vertex",
+ name: "Google Vertex",
+ models: {
+ // Same model id; must NOT produce a duplicate entry under `google`.
+ "gemini-3.1-flash-lite": {
+ id: "gemini-3.1-flash-lite",
+ name: "Gemini 3.1 Flash Lite (Vertex)",
+ reasoning: true,
+ tool_call: true,
+ release_date: "2026-05-07",
+ },
+ },
+ },
+ };
+
+ const summary = mergeModels({}, fixture);
+ const ids = summary.newEntries["google"]!.map((e) => e.model);
+ expect(ids).toEqual(["gemini-3.1-flash-lite"]);
+ expect(summary.newEntries["google"]).toHaveLength(1);
+ });
+
+ it("enforces `maxPerProvider` after deduping across mapped providers", () => {
+ const make = (id: string, date: string) => ({
+ id,
+ name: id,
+ reasoning: false,
+ tool_call: false,
+ release_date: date,
+ });
+ const fixture: ModelsDevApi = {
+ google: {
+ id: "google",
+ name: "Google",
+ models: {
+ a: make("a", "2026-01-01"),
+ b: make("b", "2026-02-01"),
+ c: make("c", "2026-03-01"),
+ },
+ },
+ "google-vertex": {
+ id: "google-vertex",
+ name: "Google Vertex",
+ models: {
+ // Overlap with `google` (a, b) + one extra unique to vertex (d).
+ a: make("a", "2026-01-01"),
+ b: make("b", "2026-02-01"),
+ d: make("d", "2026-04-01"),
+ },
+ },
+ };
+
+ const summary = mergeModels({}, fixture, { maxPerProvider: 2 });
+ expect(summary.newEntries["google"]!.map((e) => e.model)).toEqual([
+ "d", // 2026-04-01
+ "c", // 2026-03-01
+ ]);
+ });
+
+ it("forwards `cost` from models.dev when present, and omits it otherwise", () => {
+ const summary = mergeModels({}, FIXTURE_API);
+ const opus = summary.newEntries["anthropic"]!.find(
+ (e) => e.model === "claude-opus-4-7",
+ );
+ expect(opus!.cost).toEqual({ input: 15, output: 75 });
+
+ const embed = summary.newEntries["openai"]!.find(
+ (e) => e.model === "text-embedding-3-large",
+ );
+ expect(embed!.cost).toBeUndefined();
+ });
+
+ // `-n N` means "the N freshest upstream models" — we then drop ids we
+ // already have curated, rather than backfilling with the N+1th, N+2th, ...
+ // freshest "missing" model. Otherwise the cap drifts toward "always add N
+ // entries per run" instead of tracking the upstream frontier.
+ it("picks the top-N latest upstream and only appends ones we don't already have", () => {
+ const make = (id: string, date: string) => ({
+ id,
+ name: id,
+ reasoning: false,
+ tool_call: false,
+ release_date: date,
+ });
+ const fixture: ModelsDevApi = {
+ anthropic: {
+ id: "anthropic",
+ models: {
+ a1: make("a1", "2020-01-01"),
+ a2: make("a2", "2021-01-01"),
+ a3: make("a3", "2022-01-01"),
+ a4: make("a4", "2023-01-01"),
+ a5: make("a5", "2024-01-01"),
+ },
+ },
+ };
+
+ // Top 2 = {a5, a4}. We already curated a5 → only a4 is added (not a3).
+ const partial = mergeModels({ anthropic: [{ model: "a5" }] }, fixture, {
+ maxPerProvider: 2,
+ });
+ expect(partial.newEntries["anthropic"]!.map((e) => e.model)).toEqual([
+ "a4",
+ ]);
+ expect(partial.skippedExisting).toEqual(["anthropic/a5"]);
+
+ // Top 2 = {a5, a4}. Both already curated → nothing is added; we don't
+ // dredge up a3.
+ const fullyCovered = mergeModels(
+ { anthropic: [{ model: "a5" }, { model: "a4" }] },
+ fixture,
+ { maxPerProvider: 2 },
+ );
+ expect(fullyCovered.newEntries["anthropic"]).toBeUndefined();
+ expect(fullyCovered.skippedExisting.sort()).toEqual([
+ "anthropic/a4",
+ "anthropic/a5",
+ ]);
+ });
+
+ describe("cost filter", () => {
+ const make = (
+ id: string,
+ cost: { input?: number; output?: number } | undefined,
+ ) => ({
+ id,
+ name: id,
+ reasoning: false,
+ tool_call: false,
+ release_date: "2026-01-01",
+ ...(cost && { cost }),
+ });
+
+ it("pins the price ceilings", () => {
+ // Tripping these values is a deliberate product decision — bump them
+ // here and update the comment on the constants.
+ expect(MAX_COST_INPUT).toBe(30);
+ expect(MAX_COST_OUTPUT).toBe(100);
+ });
+
+ it("drops models priced at or above the input ceiling", () => {
+ const fixture: ModelsDevApi = {
+ anthropic: {
+ id: "anthropic",
+ models: {
+ "ok-below": make("ok-below", { input: 29, output: 5 }),
+ "blocked-at": make("blocked-at", { input: 30, output: 5 }),
+ "blocked-above": make("blocked-above", { input: 100, output: 5 }),
+ },
+ },
+ };
+ const summary = mergeModels({}, fixture);
+ expect(
+ summary.newEntries["anthropic"]!.map((e) => e.model).sort(),
+ ).toEqual(["ok-below"]);
+ });
+
+ it("drops models priced at or above the output ceiling", () => {
+ const fixture: ModelsDevApi = {
+ anthropic: {
+ id: "anthropic",
+ models: {
+ "ok-below": make("ok-below", { input: 5, output: 99 }),
+ "blocked-at": make("blocked-at", { input: 5, output: 100 }),
+ "blocked-above": make("blocked-above", { input: 5, output: 500 }),
+ },
+ },
+ };
+ const summary = mergeModels({}, fixture);
+ expect(
+ summary.newEntries["anthropic"]!.map((e) => e.model).sort(),
+ ).toEqual(["ok-below"]);
+ });
+
+ it("keeps models with no cost data (assume free / unknown, not frontier)", () => {
+ const fixture: ModelsDevApi = {
+ anthropic: {
+ id: "anthropic",
+ models: {
+ "no-cost": make("no-cost", undefined),
+ "partial-cost-input-only": make("partial-cost-input-only", {
+ input: 1,
+ }),
+ "partial-cost-output-only": make("partial-cost-output-only", {
+ output: 1,
+ }),
+ },
+ },
+ };
+ const summary = mergeModels({}, fixture);
+ expect(
+ summary.newEntries["anthropic"]!.map((e) => e.model).sort(),
+ ).toEqual([
+ "no-cost",
+ "partial-cost-input-only",
+ "partial-cost-output-only",
+ ]);
+ });
+
+ it("filters before the N-latest cap, so expensive models don't burn the quota", () => {
+ const fixture: ModelsDevApi = {
+ anthropic: {
+ id: "anthropic",
+ models: {
+ // Newest two are frontier-priced and should be filtered out
+ // *before* `maxPerProvider` is applied. Without pre-filtering,
+ // a cap of 2 would yield zero additions instead of two.
+ "newest-expensive": {
+ id: "newest-expensive",
+ name: "newest-expensive",
+ release_date: "2026-05-01",
+ cost: { input: 50, output: 200 },
+ },
+ "second-newest-expensive": {
+ id: "second-newest-expensive",
+ name: "second-newest-expensive",
+ release_date: "2026-04-01",
+ cost: { input: 40, output: 150 },
+ },
+ "third-newest-cheap": {
+ id: "third-newest-cheap",
+ name: "third-newest-cheap",
+ release_date: "2026-03-01",
+ cost: { input: 1, output: 5 },
+ },
+ "fourth-newest-cheap": {
+ id: "fourth-newest-cheap",
+ name: "fourth-newest-cheap",
+ release_date: "2026-02-01",
+ cost: { input: 1, output: 5 },
+ },
+ },
+ },
+ };
+ const summary = mergeModels({}, fixture, { maxPerProvider: 2 });
+ expect(summary.newEntries["anthropic"]!.map((e) => e.model)).toEqual([
+ "third-newest-cheap",
+ "fourth-newest-cheap",
+ ]);
+ });
+ });
+
+ describe("denylist", () => {
+ const make = (id: string, date = "2026-01-01") => ({
+ id,
+ name: id,
+ reasoning: false,
+ tool_call: false,
+ release_date: date,
+ });
+
+ it("seeds the denylist with gpt-5.3-codex-spark under openai", () => {
+ // Changing this set is a deliberate product decision — when a model is
+ // added or removed here, leave a comment next to it explaining why.
+ expect(MODEL_DENYLIST["openai"]?.has("gpt-5.3-codex-spark")).toBe(true);
+ });
+
+ it("drops upstream models that match the per-provider denylist", () => {
+ const fixture: ModelsDevApi = {
+ openai: {
+ id: "openai",
+ name: "OpenAI",
+ models: {
+ "gpt-5.3-codex-spark": make("gpt-5.3-codex-spark"),
+ "gpt-5.3-codex": make("gpt-5.3-codex"),
+ },
+ },
+ };
+ const summary = mergeModels({}, fixture);
+ expect(summary.newEntries["openai"]!.map((e) => e.model)).toEqual([
+ "gpt-5.3-codex",
+ ]);
+ });
+
+ it("only blocks the id under the listed provider, not everywhere", () => {
+ // A made-up `anthropic/gpt-5.3-codex-spark` would still be synced —
+ // denylist entries don't leak across providers.
+ const fixture: ModelsDevApi = {
+ anthropic: {
+ id: "anthropic",
+ name: "Anthropic",
+ models: {
+ "gpt-5.3-codex-spark": make("gpt-5.3-codex-spark"),
+ },
+ },
+ };
+ const summary = mergeModels({}, fixture);
+ expect(summary.newEntries["anthropic"]!.map((e) => e.model)).toEqual([
+ "gpt-5.3-codex-spark",
+ ]);
+ });
+
+ it("filters before the N-latest cap, so denylisted models don't burn the quota", () => {
+ const fixture: ModelsDevApi = {
+ openai: {
+ id: "openai",
+ name: "OpenAI",
+ models: {
+ // Newest is denylisted — without pre-filtering, a cap of 1 would
+ // pick this and add zero. With pre-filtering, the cap falls
+ // through to `gpt-5.3-codex`.
+ "gpt-5.3-codex-spark": make("gpt-5.3-codex-spark", "2026-05-01"),
+ "gpt-5.3-codex": make("gpt-5.3-codex", "2026-04-01"),
+ },
+ },
+ };
+ const summary = mergeModels({}, fixture, { maxPerProvider: 1 });
+ expect(summary.newEntries["openai"]!.map((e) => e.model)).toEqual([
+ "gpt-5.3-codex",
+ ]);
+ });
+ });
+});
+
+describe("syncModels", () => {
+ let tempDir: string;
+ let yamlPath: string;
+
+ beforeEach(() => {
+ tempDir = mkdtempSync(join(tmpdir(), "llm-info-sync-test-"));
+ yamlPath = join(tempDir, "models.yml");
+ writeFileSync(yamlPath, FIXTURE_YAML);
+ });
+
+ it("appends new entries to existing provider sections and creates new ones", async () => {
+ const result = await syncModels({
+ modelsYamlPath: yamlPath,
+ modelsDev: FIXTURE_API,
+ });
+
+ expect(result.added).toBe(5);
+ expect(result.preserved).toBe(1);
+
+ const onDisk = readFileSync(yamlPath, "utf-8");
+ expect(onDisk).toContain("# Manually curated section");
+ expect(onDisk).toContain("description: A hand-written description.");
+ expect(onDisk).toContain("model: claude-opus-4-7");
+ expect(onDisk).toContain("model: gpt-5-mini");
+ // Render dates as plain `YYYY-MM-DD`, not ISO timestamps.
+ expect(onDisk).toContain("release_date: 2026-04-15");
+ expect(onDisk).not.toContain("T00:00:00.000Z");
+
+ const parsed = parse(onDisk);
+ expect(Object.keys(parsed).sort()).toEqual([
+ "anthropic",
+ "azure",
+ "openai",
+ ]);
+ const opus45 = parsed.anthropic.find(
+ (m: { model: string }) => m.model === "claude-opus-4-5",
+ );
+ expect(opus45.description).toBe("A hand-written description.");
+ expect(opus45.name).toBe("Claude Opus 4.5");
+ });
+
+ it("renders flow-style arrays and a blank line between entries", async () => {
+ await syncModels({
+ modelsYamlPath: yamlPath,
+ modelsDev: FIXTURE_API,
+ });
+ const onDisk = readFileSync(yamlPath, "utf-8");
+ // No `providers:` field on entries — the parent key carries that info.
+ expect(onDisk).not.toMatch(/^\s+providers:/m);
+ // Flow style for `roles`, `capabilities`, etc.
+ expect(onDisk).toMatch(/roles: \[chat, edit\]/);
+ expect(onDisk).toMatch(/capabilities: \[thinking, tool_calling\]/);
+ // Flow-style map for `cost`.
+ expect(onDisk).toMatch(/cost: \{ ?input: 15, ?output: 75 ?\}/);
+ });
+
+ it("separates provider sections with a blank line", async () => {
+ // Fresh bootstrap so we exercise both `renderFresh` and append paths.
+ writeFileSync(yamlPath, "");
+ await syncModels({ modelsYamlPath: yamlPath, modelsDev: FIXTURE_API });
+ const fresh = readFileSync(yamlPath, "utf-8");
+ // Every top-level provider key (except the first) is preceded by a blank line.
+ expect(fresh).not.toMatch(/^\S.*\n[a-z][a-z0-9-]*:$/m);
+
+ // Re-sync against an upstream that adds a new provider — that section
+ // should also be preceded by a blank line when appended. `wandb` is in
+ // PROVIDER_MAP but not in FIXTURE_API.
+ const extended: ModelsDevApi = {
+ ...FIXTURE_API,
+ wandb: {
+ id: "wandb",
+ name: "Weights & Biases",
+ models: {
+ "deepseek-v3.1": {
+ id: "deepseek-v3.1",
+ name: "DeepSeek V3.1",
+ reasoning: false,
+ tool_call: true,
+ release_date: "2026-03-01",
+ },
+ },
+ },
+ };
+ await syncModels({ modelsYamlPath: yamlPath, modelsDev: extended });
+ const after = readFileSync(yamlPath, "utf-8");
+ expect(after).toMatch(/\n\nwandb:\n/);
+ });
+
+ it("is idempotent: running twice produces no further changes", async () => {
+ await syncModels({
+ modelsYamlPath: yamlPath,
+ modelsDev: FIXTURE_API,
+ });
+ const afterFirst = readFileSync(yamlPath, "utf-8");
+
+ const secondResult = await syncModels({
+ modelsYamlPath: yamlPath,
+ modelsDev: FIXTURE_API,
+ });
+ const afterSecond = readFileSync(yamlPath, "utf-8");
+
+ expect(secondResult.added).toBe(0);
+ expect(afterSecond).toEqual(afterFirst);
+ });
+
+ it("never overwrites a hand-curated entry, even when models.dev disagrees", async () => {
+ await syncModels({
+ modelsYamlPath: yamlPath,
+ modelsDev: FIXTURE_API,
+ });
+ const onDisk = readFileSync(yamlPath, "utf-8");
+ expect(onDisk).not.toContain("upstream-name-should-not-overwrite");
+ expect(onDisk).toContain("name: Claude Opus 4.5");
+ });
+
+ it("`mode: replace` overwrites the file with only auto-synced entries", async () => {
+ const result = await syncModels({
+ modelsYamlPath: yamlPath,
+ modelsDev: FIXTURE_API,
+ mode: "replace",
+ });
+
+ // `claude-opus-4-5` is no longer treated as existing, so it re-appears.
+ expect(result.added).toBe(6);
+ expect(result.preserved).toBe(0);
+
+ const onDisk = readFileSync(yamlPath, "utf-8");
+ expect(onDisk).not.toContain("# Manually curated section");
+ expect(onDisk).not.toContain("A hand-written description.");
+
+ const parsed = parse(onDisk);
+ const opus45 = parsed.anthropic.find(
+ (m: { model: string }) => m.model === "claude-opus-4-5",
+ );
+ expect(opus45.description).toBe("");
+ });
+
+ it("bootstraps from an empty models.yml", async () => {
+ writeFileSync(yamlPath, "");
+
+ const result = await syncModels({
+ modelsYamlPath: yamlPath,
+ modelsDev: FIXTURE_API,
+ });
+
+ expect(result.added).toBe(6);
+ expect(result.preserved).toBe(0);
+
+ const onDisk = readFileSync(yamlPath, "utf-8");
+ const parsed = parse(onDisk);
+ expect(parsed).toBeTypeOf("object");
+ expect(parsed.anthropic).toBeDefined();
+ expect(parsed.openai).toBeDefined();
+ });
+
+ it("restricts the sync to the requested providers", async () => {
+ writeFileSync(yamlPath, "");
+ const result = await syncModels({
+ modelsYamlPath: yamlPath,
+ modelsDev: FIXTURE_API,
+ providers: ["anthropic"],
+ });
+
+ expect(result.added).toBe(2); // claude-opus-4-5 + claude-opus-4-7
+ const parsed = parse(readFileSync(yamlPath, "utf-8"));
+ expect(Object.keys(parsed)).toEqual(["anthropic"]);
+ expect(
+ parsed.anthropic.map((m: { model: string }) => m.model).sort(),
+ ).toEqual(["claude-opus-4-5", "claude-opus-4-7"]);
+ });
+
+ it("preserves untouched provider sections when filtering", async () => {
+ // Start with existing entries under both anthropic and openai; sync only
+ // anthropic and confirm openai is left alone.
+ const seed = `anthropic:
+ - name: Existing Anthropic
+ model: claude-existing
+ description: hand-curated
+ roles: [chat, edit]
+ capabilities: []
+ input_types: []
+ output_types: []
+ release_date: 2024-01-01
+
+openai:
+ - name: Existing OpenAI
+ model: gpt-existing
+ description: hand-curated
+ roles: [chat, edit]
+ capabilities: []
+ input_types: []
+ output_types: []
+ release_date: 2024-01-01
+`;
+ writeFileSync(yamlPath, seed);
+
+ const result = await syncModels({
+ modelsYamlPath: yamlPath,
+ modelsDev: FIXTURE_API,
+ providers: ["anthropic"],
+ });
+
+ // openai entries are not counted as preserved (out of filter scope).
+ expect(result.preserved).toBe(1);
+ expect(result.added).toBe(2);
+
+ const onDisk = readFileSync(yamlPath, "utf-8");
+ expect(onDisk).toContain("model: gpt-existing"); // openai section untouched
+ expect(onDisk).not.toContain("model: gpt-5-mini"); // openai sync skipped
+ expect(onDisk).toContain("model: claude-opus-4-7"); // anthropic synced
+ });
+
+ it("threads `maxPerProvider` through to the merge", async () => {
+ writeFileSync(yamlPath, "");
+ const result = await syncModels({
+ modelsYamlPath: yamlPath,
+ modelsDev: FIXTURE_API,
+ maxPerProvider: 1,
+ });
+
+ // With 1 per provider: anthropic gets 1, openai gets 1, azure gets 1.
+ // claude-opus-4-5 (preserved earlier) doesn't apply here since the file
+ // starts empty.
+ expect(result.added).toBe(3);
+ const parsed = parse(readFileSync(yamlPath, "utf-8"));
+ expect(parsed.anthropic).toHaveLength(1);
+ expect(parsed.openai).toHaveLength(1);
+ expect(parsed.azure).toHaveLength(1);
+ });
+
+ it("does not write the file when there are no new entries", async () => {
+ const full = `anthropic:
+ - name: Claude Opus 4.5
+ model: claude-opus-4-5
+ - name: Claude Opus 4.7
+ model: claude-opus-4-7
+openai:
+ - name: GPT-5 Mini
+ model: gpt-5-mini
+ - name: GPT-5.5
+ model: gpt-5.5
+ - name: OpenAI Text Embedding 3 Large
+ model: text-embedding-3-large
+azure:
+ - name: GPT-5.5
+ model: gpt-5.5
+`;
+ writeFileSync(yamlPath, full);
+ const before = readFileSync(yamlPath, "utf-8");
+
+ const result = await syncModels({
+ modelsYamlPath: yamlPath,
+ modelsDev: FIXTURE_API,
+ });
+
+ const after = readFileSync(yamlPath, "utf-8");
+ expect(result.added).toBe(0);
+ expect(after).toEqual(before);
+ });
+});
diff --git a/packages/llm-info/src/cli.ts b/packages/llm-info/src/cli.ts
new file mode 100644
index 0000000000..4540744383
--- /dev/null
+++ b/packages/llm-info/src/cli.ts
@@ -0,0 +1,106 @@
+/* Copyright 2026 Marimo. All rights reserved. */
+
+import { Logger } from "./simple_logger.ts";
+import { PROVIDER_MAP } from "./sources/merge.ts";
+import type { SyncMode } from "./sync-models.ts";
+
+export interface CliArgs {
+ mode: SyncMode;
+ maxPerProvider?: number;
+ providers?: string[];
+}
+
+/** Read `--name=value`, `--name value`, or any alias of those, from argv. */
+function getFlag(
+ argv: readonly string[],
+ names: readonly string[],
+): string | undefined {
+ for (let i = 0; i < argv.length; i++) {
+ const arg = argv[i];
+ for (const name of names) {
+ if (arg.startsWith(`${name}=`)) {
+ return arg.slice(name.length + 1);
+ }
+ if (arg === name && i + 1 < argv.length) {
+ return argv[i + 1];
+ }
+ }
+ }
+ return undefined;
+}
+
+function parseMode(argv: readonly string[]): SyncMode {
+ if (argv.includes("--replace") || argv.includes("-r")) {
+ return "replace";
+ }
+ // Accept both `--mode=replace` and the spaced form `--mode replace`.
+ const raw = getFlag(argv, ["--mode"]);
+ if (raw === "replace") {
+ return "replace";
+ }
+ if (raw !== undefined && raw !== "append") {
+ throw new Error(
+ `Unknown --mode value "${raw}" — expected "append" or "replace".`,
+ );
+ }
+ return "append";
+}
+
+function parseMaxPerProvider(argv: readonly string[]): number | undefined {
+ const raw = getFlag(argv, ["--max-per-provider", "--max", "-n"]);
+ if (raw === undefined) {
+ return undefined;
+ }
+ const n = Number(raw);
+ return Number.isFinite(n) && n > 0 ? n : undefined;
+}
+
+function parseProviders(argv: readonly string[]): string[] | undefined {
+ const raw = getFlag(argv, ["--provider", "--providers", "-p"]);
+ if (raw === undefined) {
+ return undefined;
+ }
+ const known: ReadonlySet<string> = new Set(Object.values(PROVIDER_MAP));
+ const knownList = [...known].sort().join(", ");
+ const requested = raw
+ .split(",")
+ .map((s) => s?.trim())
+ .filter((s): s is string => Boolean(s));
+
+ if (requested.length === 0) {
+ throw new Error(
+ `--provider was given but no value parsed. Known providers: ${knownList}`,
+ );
+ }
+
+ const valid: string[] = [];
+ const invalid: string[] = [];
+ for (const id of requested) {
+ if (known.has(id)) {
+ valid.push(id);
+ } else {
+ invalid.push(id);
+ }
+ }
+ if (invalid.length > 0) {
+ Logger.warn(
+ `Unknown provider(s) ignored: ${invalid.join(", ")} — known providers: ${knownList}`,
+ );
+ }
+ // Refuse to proceed with an empty filter — under --replace this would
+ // silently wipe `models.yml`. Caller must pass at least one known provider.
+ if (valid.length === 0) {
+ throw new Error(
+ `No known providers in --provider="${raw}". Known providers: ${knownList}`,
+ );
+ }
+ return valid;
+}
+
+export function parseCliArgs(argv: readonly string[]): CliArgs {
+ return {
+ mode: parseMode(argv),
+ maxPerProvider: parseMaxPerProvider(argv),
+ providers: parseProviders(argv),
+ };
+}
diff --git a/packages/llm-info/src/generate.ts b/packages/llm-info/src/generate.ts
index bc8ab7dcc3..2bee3a7f76 100644
--- a/packages/llm-info/src/generate.ts
+++ b/packages/llm-info/src/generate.ts
@@ -8,16 +8,44 @@ import { z } from "zod";
import { Logger } from "./simple_logger.ts";
const ROLES = ["chat", "edit", "rerank", "embed", "autocomplete"] as const;
+const CAPABILITIES = ["thinking", "tool_calling"] as const;
+const DATA_TYPES = ["text", "image", "pdf"] as const;
+
+const CostSchema = z
+ .object({
+ input: z.number().optional(),
+ output: z.number().optional(),
+ })
+ .partial();
+
+/** YAML may parse `YYYY-MM-DD` scalars as Date; coerce back to ISO string. */
+const ReleaseDateSchema = z
+ .union([z.string(), z.date()])
+ .optional()
+ .transform((v) => {
+ if (v === undefined) return undefined;
+ if (v instanceof Date) return v.toISOString().slice(0, 10);
+ return v;
+ });
export const LLMInfoSchema = z.object({
name: z.string(),
model: z.string(),
description: z.string(),
- providers: z.array(z.string()),
roles: z.array(z.enum(ROLES)),
- thinking: z.boolean().default(false),
+ capabilities: z.array(z.enum(CAPABILITIES)).default([]),
+ input_types: z.array(z.enum(DATA_TYPES)).default([]),
+ output_types: z.array(z.enum(DATA_TYPES)).default([]),
+ release_date: ReleaseDateSchema,
+ cost: CostSchema.optional(),
});
+/** Top-level shape of `models.yml` / `models.json`: provider id → models. */
+export const ModelsByProviderSchema = z.record(
+ z.string(),
+ z.array(LLMInfoSchema),
+);
+
export const ProviderSchema = z.object({
name: z.string(),
id: z.string(),
@@ -33,7 +61,6 @@ function ensureDirectoryExists(filePath: string): void {
try {
mkdirSync(dir, { recursive: true });
} catch (error: any) {
- // Ignore error if directory already exists, otherwise rethrow
if (error?.code !== "EEXIST") {
Logger.error("Failed to create directory:", error);
throw error;
@@ -41,30 +68,25 @@ function ensureDirectoryExists(filePath: string): void {
}
}
-function loadAndValidateModels(yamlPath: string): any[] {
+function loadAndValidateModels(yamlPath: string): Record<string, unknown[]> {
const yamlContent = readFileSync(yamlPath, "utf-8");
- const models = parse(yamlContent);
+ const raw = parse(yamlContent);
- if (!Array.isArray(models)) {
- throw new Error("Models YAML file must contain an array");
+ if (raw == null || typeof raw !== "object" || Array.isArray(raw)) {
+ throw new Error(
+ "Models YAML must be a map keyed by provider id (e.g. `anthropic:`)",
+ );
}
- // Validate each model against the schema
- const validatedModels = models.map((model, index) => {
- try {
- return LLMInfoSchema.parse(model);
- } catch (error) {
- Logger.error(`Validation failed for model at index ${index}:`, error);
- throw new Error(
- `Model validation failed at index ${index}: ${JSON.stringify(model, null, 2)}`,
- );
- }
- });
-
- return validatedModels;
+ try {
+ return ModelsByProviderSchema.parse(raw);
+ } catch (error) {
+ Logger.error("Model validation failed:", error);
+ throw error;
+ }
}
-function loadAndValidateProviders(yamlPath: string): any[] {
+function loadAndValidateProviders(yamlPath: string): unknown[] {
const yamlContent = readFileSync(yamlPath, "utf-8");
const providers = parse(yamlContent);
@@ -72,8 +94,7 @@ function loadAndValidateProviders(yamlPath: string): any[] {
throw new Error("Providers YAML file must contain an array");
}
- // Validate each provider against the schema
- const validatedProviders = providers.map((provider, index) => {
+ return providers.map((provider, index) => {
try {
return ProviderSchema.parse(provider);
} catch (error) {
@@ -83,18 +104,15 @@ function loadAndValidateProviders(yamlPath: string): any[] {
);
}
});
-
- return validatedProviders;
}
-function writeJsonFile(filePath: string, data: any): void {
+function writeJsonFile(filePath: string, data: unknown): void {
ensureDirectoryExists(filePath);
writeFileSync(filePath, JSON.stringify(data, null, 2));
}
async function main(): Promise<void> {
try {
- // Define paths
const dataDir = join(__dirname, "../data");
const generatedDir = join(dataDir, "generated");
const modelsYamlPath = join(dataDir, "models.yml");
@@ -102,20 +120,18 @@ async function main(): Promise<void> {
const modelsJsonPath = join(generatedDir, "models.json");
const providersJsonPath = join(generatedDir, "providers.json");
- // For compatibility with Vite and other bundlers, `import` returns a JS module and not a JSON object.
- // So we need to nest the models and providers data under a json key to access them,
- // otherwise a keyword can conflict with a JS reserved keyword (e.g. `default` or `with`).
-
- // Load and validate models
const models = loadAndValidateModels(modelsYamlPath);
- writeJsonFile(modelsJsonPath, { models: models });
+ writeJsonFile(modelsJsonPath, { models });
- // Load and validate providers
const providers = loadAndValidateProviders(providersYamlPath);
- writeJsonFile(providersJsonPath, { providers: providers });
+ writeJsonFile(providersJsonPath, { providers });
+ const totalModels = Object.values(models).reduce(
+ (acc, list) => acc + list.length,
+ 0,
+ );
Logger.info(
- `Generated ${models.length} models and ${providers.length} providers`,
+ `Generated ${totalModels} models across ${Object.keys(models).length} providers and ${providers.length} provider entries`,
);
} catch (error) {
Logger.error("Generation failed:", error);
@@ -123,5 +139,4 @@ async function main(): Promise<void> {
}
}
-// Run the script
main().catch(Logger.error);
diff --git a/packages/llm-info/src/index.ts b/packages/llm-info/src/index.ts
index 730035e536..00546c54ac 100644
--- a/packages/llm-info/src/index.ts
+++ b/packages/llm-info/src/index.ts
@@ -7,15 +7,49 @@ export const ROLES = [
] as const;
export type Role = (typeof ROLES)[number];
+type Capability = "thinking" | "tool_calling";
+type DataType = "text" | "image" | "pdf";
+
+/**
+ * Per-token pricing in USD per 1M tokens, mirroring the `cost` block on
+ * `models.dev`. All fields are optional — open-weights / self-hosted models
+ * usually have none.
+ */
+export interface AiModelCost {
+ input?: number;
+ output?: number;
+}
+
export interface AiModel {
name: string;
model: string;
description: string;
- providers: string[];
roles: Role[];
- thinking: boolean;
+ capabilities: Capability[];
+ input_types: DataType[];
+ output_types: DataType[];
+ /** ISO `YYYY-MM-DD` — stored as a string to round-trip through YAML/JSON. */
+ release_date: string;
+ cost?: AiModelCost;
}
+export type SyncableProviderId =
+ | "anthropic"
+ | "openai"
+ | "google"
+ | "bedrock"
+ | "azure"
+ | "github"
+ | "ollama"
+ | "wandb"
+ | "opencode-go";
+
+/**
+ * `models.yml` and `models.json` are keyed by provider id at the top level —
+ * the same model may appear under multiple providers as independent entries.
+ */
+export type ModelsByProvider = Partial<Record<SyncableProviderId, AiModel[]>>;
+
export interface AiProvider {
name: string;
id: string;
diff --git a/packages/llm-info/src/sources/merge.ts b/packages/llm-info/src/sources/merge.ts
new file mode 100644
index 0000000000..0f311533ff
--- /dev/null
+++ b/packages/llm-info/src/sources/merge.ts
@@ -0,0 +1,283 @@
+/* Copyright 2026 Marimo. All rights reserved. */
+
+import type {
+ AiModel,
+ ModelsByProvider,
+ Role,
+ SyncableProviderId,
+} from "../index.ts";
+import type { ModelsDevApi, ModelsDevModel } from "./models-dev.ts";
+
+/**
+ * Allowlist of models.dev provider ids → marimo provider ids. Models from
+ * providers outside this map are ignored. To opt a provider in, add it to
+ * `providers.yml` and register it here.
+ */
+export const PROVIDER_MAP = {
+ anthropic: "anthropic",
+ openai: "openai",
+ google: "google",
+ "google-vertex": "google",
+ "amazon-bedrock": "bedrock",
+ azure: "azure",
+ "github-models": "github",
+ "ollama-cloud": "ollama",
+ wandb: "wandb",
+ "opencode-go": "opencode-go",
+} as const satisfies Readonly<Record<string, SyncableProviderId>>;
+
+const MODALITIES = new Set(["text", "image", "pdf"]);
+
+/** Sentinel for models with no `release_date` upstream — sorts to the bottom. */
+const UNKNOWN_DATE = "1970-01-01";
+export const MAX_MODELS_PER_PROVIDER = 10;
+
+/**
+ * Frontier-tier price ceilings, in USD per million tokens.
+ */
+export const MAX_COST_INPUT = 30;
+export const MAX_COST_OUTPUT = 100;
+
+/**
+ * Per-provider explicit denylist of model ids.
+
+ * Note: this only blocks future appends. To remove a model that's already
+ * in `models.yml`, delete the entry by hand.
+ */
+export const MODEL_DENYLIST: Readonly<Record<string, ReadonlySet<string>>> = {
+ openai: new Set([
+ // Surfaced inside ChatGPT only; not exposed via the OpenAI API.
+ "gpt-5.3-codex-spark",
+ ]),
+};
+
+function isDenylisted(marimoProviderId: string, modelId: string): boolean {
+ return MODEL_DENYLIST[marimoProviderId]?.has(modelId) ?? false;
+}
+
+/**
+ * Loose shape for entries read from `models.yml` — we only need the `model` id
+ * to detect duplicates within a provider section.
+ */
+export type ExistingEntry = Readonly<{ model: string }>;
+export type ExistingByProvider = Readonly<
+ Record<string, readonly ExistingEntry[]>
+>;
+
+export interface MergeSummary {
+ /** New entries grouped by provider id, ready to merge into `models.yml`. */
+ newEntries: ModelsByProvider;
+ preservedCount: number;
+ /** `provider/model` ids skipped because they already exist locally. */
+ skippedExisting: string[];
+}
+
+function deriveRoles(model: ModelsDevModel): Role[] {
+ const id = model.id.toLowerCase();
+ const name = model.name.toLowerCase();
+ if (id.includes("embed") || name.includes("embedding")) {
+ return ["embed"];
+ }
+ if (id.includes("rerank") || name.includes("rerank")) {
+ return ["rerank"];
+ }
+ return ["chat", "edit"];
+}
+
+function deriveCapabilities(model: ModelsDevModel): AiModel["capabilities"] {
+ const capabilities: AiModel["capabilities"] = [];
+ if (model.reasoning) {
+ capabilities.push("thinking");
+ }
+ if (model.tool_call) {
+ capabilities.push("tool_calling");
+ }
+ return capabilities;
+}
+
+function filterModalities(
+ modalities: readonly string[] | undefined,
+): AiModel["input_types"] {
+ if (!modalities) {
+ return [];
+ }
+ return modalities.filter((m): m is "text" | "image" | "pdf" =>
+ MODALITIES.has(m),
+ );
+}
+
+/**
+ * Normalize upstream `release_date` to `YYYY-MM-DD`. Accepts shorthand
+ * (`YYYY-MM`), full ISO timestamps, and anything else `Date.parse` understands
+ * — but always emits the canonical `YYYY-MM-DD` form so downstream lex-sort
+ * and the `AiModel.release_date` type contract hold. Unparsable / missing
+ * values fall back to the sentinel.
+ */
+function parseReleaseDate(raw: string | undefined): string {
+ if (!raw) {
+ return UNKNOWN_DATE;
+ }
+ // Pad `YYYY-MM` shorthand to first of month before parsing.
+ const padded = /^\d{4}-\d{2}$/.test(raw) ? `${raw}-01` : raw;
+ const ms = Date.parse(padded);
+ if (Number.isNaN(ms)) {
+ return UNKNOWN_DATE;
+ }
+ return new Date(ms).toISOString().slice(0, 10);
+}
+
+function isWithinCostBudget(source: ModelsDevModel): boolean {
+ const cost = source.cost ?? {};
+ if (typeof cost.input === "number" && cost.input >= MAX_COST_INPUT) {
+ return false;
+ }
+ if (typeof cost.output === "number" && cost.output >= MAX_COST_OUTPUT) {
+ return false;
+ }
+ return true;
+}
+
+function deriveCost(source: ModelsDevModel): AiModel["cost"] {
+ const { input, output } = source.cost ?? {};
+ if (typeof input !== "number" && typeof output !== "number") {
+ return undefined;
+ }
+ return {
+ ...(typeof input === "number" && { input }),
+ ...(typeof output === "number" && { output }),
+ };
+}
+
+function buildAiModel(source: ModelsDevModel): AiModel {
+ const cost = deriveCost(source);
+ return {
+ name: source.name,
+ model: source.id,
+ description: "",
+ roles: deriveRoles(source),
+ capabilities: deriveCapabilities(source),
+ input_types: filterModalities(source.modalities?.input),
+ output_types: filterModalities(source.modalities?.output),
+ release_date: parseReleaseDate(source.release_date),
+ ...(cost && { cost }),
+ };
+}
+
+/**
+ * Sort newest-first by `release_date` and trim to `maxPerProvider`. Tie-break
+ * on model id so identical dates produce a deterministic order.
+ */
+function sortAndTrim(entries: AiModel[], maxPerProvider: number): AiModel[] {
+ const sorted = [...entries].sort((a, b) => {
+ // ISO `YYYY-MM-DD` strings sort lexicographically the same as by date.
+ if (a.release_date !== b.release_date) {
+ return a.release_date < b.release_date ? 1 : -1;
+ }
+ return a.model.localeCompare(b.model);
+ });
+ return sorted.slice(0, maxPerProvider);
+}
+
+export interface MergeOptions {
+ /** Defaults to {@link MAX_MODELS_PER_PROVIDER}. */
+ maxPerProvider?: number;
+ /**
+ * Restrict the sync to these marimo provider ids. Defaults to all providers
+ * in {@link PROVIDER_MAP}.
+ */
+ providers?: readonly string[];
+}
+
+/**
+ * Merge models.dev data into existing per-provider entries.
+ *
+ * Pipeline per provider:
+ * 1. Collect upstream candidates, dropping anything above the cost ceilings
+ * or explicitly listed in `MODEL_DENYLIST`.
+ * 2. Sort newest-first and trim to `maxPerProvider` — this is what `-n N`
+ * means: "the N freshest models upstream is exposing right now."
+ * 3. Of that top-N, drop ids we already have locally; the remainder is what
+ * gets appended.
+ *
+ * Consequences worth knowing:
+ * - Existing entries (matched by `(provider, model)` id pair) are never
+ * modified — all human curation is preserved.
+ * - New entries are added with `description: ""` for the human to fill in.
+ * - Idempotent: a second run with the same inputs produces no new entries.
+ * - If every model in the top-N is already curated, this run adds zero —
+ * we don't backfill with older "missing" models.
+ */
+export function mergeModels(
+ existing: ExistingByProvider,
+ modelsDev: ModelsDevApi,
+ options: MergeOptions = {},
+): MergeSummary {
+ const { maxPerProvider = MAX_MODELS_PER_PROVIDER, providers } = options;
+ const providerFilter = providers ? new Set(providers) : null;
+ const skippedExisting: string[] = [];
+ let preservedCount = 0;
+
+ for (const [provider, entries] of Object.entries(existing)) {
+ if (providerFilter && !providerFilter.has(provider)) {
+ continue;
+ }
+ preservedCount += entries.length;
+ }
+
+ // Multiple models.dev provider ids can map to the same marimo provider
+ // (e.g. `google` + `google-vertex` → `google`). Accumulate candidates per
+ // marimo provider id, de-duplicating by model id, before sorting + trimming.
+ const candidatesByProvider = new Map<string, Map<string, AiModel>>();
+
+ for (const [devProviderId, marimoProviderId] of Object.entries(
+ PROVIDER_MAP,
+ )) {
+ if (providerFilter && !providerFilter.has(marimoProviderId)) {
+ continue;
+ }
+ const provider = modelsDev[devProviderId];
+ if (!provider) {
+ continue;
+ }
+ const bucket =
+ candidatesByProvider.get(marimoProviderId) ?? new Map<string, AiModel>();
+
+ for (const model of Object.values(provider.models)) {
+ if (!isWithinCostBudget(model)) {
+ continue;
+ }
+ if (isDenylisted(marimoProviderId, model.id)) {
+ continue;
+ }
+ // First mapped source wins on conflict — later iterations skip dupes.
+ if (!bucket.has(model.id)) {
+ bucket.set(model.id, buildAiModel(model));
+ }
+ }
+ candidatesByProvider.set(marimoProviderId, bucket);
+ }
+
+ const newEntries: ModelsByProvider = {};
+ for (const [marimoProviderId, bucket] of candidatesByProvider) {
+ if (bucket.size === 0) {
+ continue;
+ }
+ const topN = sortAndTrim([...bucket.values()], maxPerProvider);
+ const existingIds = new Set(
+ (existing[marimoProviderId] ?? []).map((m) => m.model),
+ );
+ const fresh: AiModel[] = [];
+ for (const entry of topN) {
+ if (existingIds.has(entry.model)) {
+ skippedExisting.push(`${marimoProviderId}/${entry.model}`);
+ continue;
+ }
+ fresh.push(entry);
+ }
+ if (fresh.length > 0) {
+ newEntries[marimoProviderId as SyncableProviderId] = fresh;
+ }
+ }
+
+ return { newEntries, preservedCount, skippedExisting };
+}
diff --git a/packages/llm-info/src/sources/models-dev.ts b/packages/llm-info/src/sources/models-dev.ts
new file mode 100644
index 0000000000..d7f6e7691b
--- /dev/null
+++ b/packages/llm-info/src/sources/models-dev.ts
@@ -0,0 +1,122 @@
+/* Copyright 2026 Marimo. All rights reserved. */
+
+import { z } from "zod";
+import { Logger } from "../simple_logger.ts";
+
+export const ModelsDevModelSchema = z.object({
+ id: z.string(),
+ name: z.string(),
+ reasoning: z.boolean().optional(),
+ tool_call: z.boolean().optional(),
+ release_date: z.string().optional(),
+ modalities: z
+ .object({
+ input: z.array(z.string()).optional(),
+ output: z.array(z.string()).optional(),
+ })
+ .optional(),
+ limit: z
+ .object({
+ context: z.number().optional(),
+ output: z.number().optional(),
+ })
+ .optional(),
+ cost: z
+ .object({
+ input: z.number().optional(),
+ output: z.number().optional(),
+ })
+ .partial()
+ .optional(),
+});
+
+export type ModelsDevModel = z.infer<typeof ModelsDevModelSchema>;
+
+export interface ModelsDevProvider {
+ id: string;
+ name?: string;
+ models: Record<string, ModelsDevModel>;
+}
+
+export type ModelsDevApi = Record<string, ModelsDevProvider>;
+
+const DEFAULT_URL = "https://models.dev/api.json";
+
+export function parseModelsDev(raw: unknown): ModelsDevApi {
+ if (typeof raw !== "object" || raw === null || Array.isArray(raw)) {
+ throw new Error("models.dev response is not an object");
+ }
+
+ const out: ModelsDevApi = {};
+ let skippedModels = 0;
+
+ for (const [providerId, providerRaw] of Object.entries(
+ raw as Record<string, unknown>,
+ )) {
+ if (
+ typeof providerRaw !== "object" ||
+ providerRaw === null ||
+ Array.isArray(providerRaw)
+ ) {
+ Logger.warn(
+ `models.dev: skipping provider "${providerId}" (not an object)`,
+ );
+ continue;
+ }
+ const providerObj = providerRaw as {
+ id?: unknown;
+ name?: unknown;
+ models?: unknown;
+ };
+ if (
+ typeof providerObj.models !== "object" ||
+ providerObj.models === null ||
+ Array.isArray(providerObj.models)
+ ) {
+ Logger.warn(`models.dev: skipping provider "${providerId}" (no models)`);
+ continue;
+ }
+
+ const models: Record<string, ModelsDevModel> = {};
+ for (const [modelId, modelRaw] of Object.entries(
+ providerObj.models as Record<string, unknown>,
+ )) {
+ const parsed = ModelsDevModelSchema.safeParse(modelRaw);
+ if (parsed.success) {
+ models[modelId] = parsed.data;
+ } else {
+ skippedModels += 1;
+ Logger.warn(
+ `models.dev: skipping ${providerId}/${modelId} (${parsed.error.issues[0]?.message ?? "validation error"})`,
+ );
+ }
+ }
+
+ out[providerId] = {
+ id: typeof providerObj.id === "string" ? providerObj.id : providerId,
+ name: typeof providerObj.name === "string" ? providerObj.name : undefined,
+ models,
+ };
+ }
+
+ if (skippedModels > 0) {
+ Logger.warn(
+ `models.dev: skipped ${skippedModels} malformed model(s) total`,
+ );
+ }
+
+ return out;
+}
+
+export async function fetchModelsDev(
+ url: string = DEFAULT_URL,
+): Promise<ModelsDevApi> {
+ const response = await fetch(url);
+ if (!response.ok) {
+ throw new Error(
+ `Failed to fetch ${url}: ${response.status} ${response.statusText}`,
+ );
+ }
+ const json = (await response.json()) as unknown;
+ return parseModelsDev(json);
+}
diff --git a/packages/llm-info/src/sync-models.ts b/packages/llm-info/src/sync-models.ts
new file mode 100644
index 0000000000..5855c8114d
--- /dev/null
+++ b/packages/llm-info/src/sync-models.ts
@@ -0,0 +1,307 @@
+#!/usr/bin/env node
+/* Copyright 2026 Marimo. All rights reserved. */
+
+import { readFileSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+import {
+ Document,
+ isMap,
+ isSeq,
+ parseDocument,
+ type YAMLMap,
+ type YAMLSeq,
+} from "yaml";
+import { parseCliArgs } from "./cli.ts";
+import type { AiModel, ModelsByProvider } from "./index.ts";
+import { Logger } from "./simple_logger.ts";
+import {
+ type ExistingByProvider,
+ type ExistingEntry,
+ MAX_MODELS_PER_PROVIDER,
+ mergeModels,
+} from "./sources/merge.ts";
+import { fetchModelsDev, type ModelsDevApi } from "./sources/models-dev.ts";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+/**
+ * - `append`: keep existing entries, add only what's new (default).
+ * - `replace`: overwrite the file with a fresh sync — destructive, for
+ * bootstrapping or regenerating from scratch.
+ */
+export type SyncMode = "append" | "replace";
+
+interface SyncOptions {
+ modelsYamlPath: string;
+ /** Pre-loaded api.json for testing; otherwise fetched live. */
+ modelsDev?: ModelsDevApi;
+ write?: boolean;
+ mode?: SyncMode;
+ /** Cap on new entries appended per provider section. */
+ maxPerProvider?: number;
+ /** Restrict sync to these marimo provider ids (defaults to all). */
+ providers?: readonly string[];
+}
+
+export interface SyncResult {
+ added: number;
+ preserved: number;
+ yaml: string;
+}
+
+/** Sequence keys rendered in flow style (`[a, b]`) to match the existing file. */
+const FLOW_SEQ_KEYS = new Set([
+ "roles",
+ "capabilities",
+ "input_types",
+ "output_types",
+]);
+
+/** Map keys rendered in flow style (`{a: 1, b: 2}`). */
+const FLOW_MAP_KEYS = new Set(["cost"]);
+
+/** YAML serializes `Date` as an ISO timestamp; we want plain `YYYY-MM-DD`. */
+function flattenDates(entry: AiModel): Record<string, unknown> {
+ const out: Record<string, unknown> = { ...entry };
+ for (const [key, value] of Object.entries(out)) {
+ if (value instanceof Date) {
+ out[key] = value.toISOString().slice(0, 10);
+ }
+ }
+ return out;
+}
+
+function parseExistingModels(yamlText: string): ExistingByProvider {
+ const doc = parseDocument(yamlText);
+ if (doc.contents == null) {
+ return {};
+ }
+ if (!isMap(doc.contents)) {
+ throw new Error(
+ "Expected a map at the root of models.yml (keyed by provider)",
+ );
+ }
+ const result: Record<string, ExistingEntry[]> = {};
+ for (const pair of (doc.contents as YAMLMap).items) {
+ const provider = (pair.key as { value?: unknown })?.value;
+ if (typeof provider !== "string") {
+ continue;
+ }
+ if (!isSeq(pair.value)) {
+ result[provider] = [];
+ continue;
+ }
+ const entries: ExistingEntry[] = [];
+ for (const item of (pair.value as YAMLSeq).items) {
+ if (!isMap(item)) {
+ continue;
+ }
+ for (const subPair of (item as YAMLMap).items) {
+ const key = (subPair.key as { value?: unknown })?.value;
+ const value = (subPair.value as { value?: unknown })?.value;
+ if (key === "model" && typeof value === "string") {
+ entries.push({ model: value });
+ break;
+ }
+ }
+ }
+ result[provider] = entries;
+ }
+ return result;
+}
+
+/**
+ * Build a `YAMLMap` for one entry with the right flow-style array fields.
+ */
+function buildEntryNode(doc: Document, entry: AiModel): YAMLMap {
+ const node = doc.createNode(flattenDates(entry)) as YAMLMap;
+ for (const pair of node.items) {
+ const key = (pair.key as { value?: unknown })?.value;
+ if (typeof key !== "string") {
+ continue;
+ }
+ if (FLOW_SEQ_KEYS.has(key) && isSeq(pair.value)) {
+ pair.value.flow = true;
+ } else if (FLOW_MAP_KEYS.has(key) && isMap(pair.value)) {
+ pair.value.flow = true;
+ }
+ }
+ return node;
+}
+
+/**
+ * Add a `provider: [...]` section to the root map, with a blank line before
+ * the key when `spaceBefore` is true (used to separate provider sections).
+ */
+function addProviderSection(
+ doc: Document,
+ root: YAMLMap,
+ provider: string,
+ seq: YAMLSeq,
+ spaceBefore: boolean,
+): void {
+ const keyNode = doc.createNode(provider);
+ if (spaceBefore) {
+ (keyNode as { spaceBefore?: boolean }).spaceBefore = true;
+ }
+ root.add({ key: keyNode, value: seq });
+}
+
+/**
+ * Render a fresh `models.yml` from scratch (replace mode or empty bootstrap).
+ */
+function renderFresh(entries: ModelsByProvider): string {
+ const doc = new Document({});
+ const root = doc.contents as YAMLMap;
+ for (const [i, [provider, models]] of Object.entries(entries).entries()) {
+ const seq = doc.createNode([]) as YAMLSeq;
+ for (const [j, model] of models.entries()) {
+ const item = buildEntryNode(doc, model);
+ if (j > 0) {
+ (item as { spaceBefore?: boolean }).spaceBefore = true;
+ }
+ seq.items.push(item);
+ }
+ addProviderSection(doc, root, provider, seq, i > 0);
+ }
+ return doc.toString({ lineWidth: 0, flowCollectionPadding: false });
+}
+
+/**
+ * Append new entries into an existing document, creating provider sections if
+ * needed. Preserves comments and ordering of unchanged sections.
+ */
+function appendIntoDocument(
+ yamlText: string,
+ newEntries: ModelsByProvider,
+): string {
+ const doc = parseDocument(yamlText);
+ if (doc.contents == null) {
+ // Bootstrap an empty map at the root.
+ doc.contents = doc.createNode({}) as unknown as typeof doc.contents;
+ }
+ if (!isMap(doc.contents)) {
+ throw new Error(
+ "Expected a map at the root of models.yml (keyed by provider)",
+ );
+ }
+ const root = doc.contents as unknown as YAMLMap;
+
+ for (const [provider, models] of Object.entries(newEntries)) {
+ if (models.length === 0) {
+ continue;
+ }
+ let seq = findProviderSeq(root, provider);
+ if (!seq) {
+ seq = doc.createNode([]) as YAMLSeq;
+ // New section appended after existing content — always blank-line separated.
+ addProviderSection(doc, root, provider, seq, true);
+ }
+ for (const model of models) {
+ const item = buildEntryNode(doc, model);
+ if (seq.items.length > 0) {
+ (item as { spaceBefore?: boolean }).spaceBefore = true;
+ }
+ seq.items.push(item);
+ }
+ }
+
+ return doc.toString({ lineWidth: 0, flowCollectionPadding: false });
+}
+
+function findProviderSeq(root: YAMLMap, provider: string): YAMLSeq | null {
+ for (const pair of root.items) {
+ const key = (pair.key as { value?: unknown })?.value;
+ if (key === provider && isSeq(pair.value)) {
+ return pair.value;
+ }
+ }
+ return null;
+}
+
+function countEntries(entries: ModelsByProvider): number {
+ let total = 0;
+ for (const list of Object.values(entries)) {
+ total += list.length;
+ }
+ return total;
+}
+
+export async function syncModels(options: SyncOptions): Promise<SyncResult> {
+ const {
+ modelsYamlPath,
+ write = true,
+ mode = "append",
+ maxPerProvider,
+ providers,
+ } = options;
+ const modelsDev = options.modelsDev ?? (await fetchModelsDev());
+
+ // `replace` mode pretends the file is empty so everything is treated as new.
+ const existingText =
+ mode === "replace" ? "" : readFileSync(modelsYamlPath, "utf-8");
+ const existing = parseExistingModels(existingText);
+ const summary = mergeModels(existing, modelsDev, {
+ maxPerProvider,
+ providers,
+ });
+
+ const addedCount = countEntries(summary.newEntries);
+ const isFresh = mode === "replace" || existingText.trim() === "";
+ const yaml = isFresh
+ ? renderFresh(summary.newEntries)
+ : addedCount > 0
+ ? appendIntoDocument(existingText, summary.newEntries)
+ : existingText;
+
+ if (write && (addedCount > 0 || mode === "replace")) {
+ writeFileSync(modelsYamlPath, yaml);
+ }
+
+ return {
+ added: addedCount,
+ preserved: summary.preservedCount,
+ yaml,
+ };
+}
+
+async function main(): Promise<void> {
+ try {
+ const dataDir = join(__dirname, "../data");
+ const args = parseCliArgs(process.argv.slice(2));
+ const max = args.maxPerProvider ?? MAX_MODELS_PER_PROVIDER;
+ const providers = args.providers?.join(",") ?? "all";
+ Logger.info(
+ `Fetching models.dev catalog (mode: ${args.mode}, max-per-provider: ${max}, providers: ${providers})...`,
+ );
+
+ const result = await syncModels({
+ ...args,
+ modelsYamlPath: join(dataDir, "models.yml"),
+ });
+
+ Logger.info(
+ `Sync complete: added ${result.added} new model(s), preserved ${result.preserved} existing entries.`,
+ );
+ Logger.info(
+ result.added > 0
+ ? "Review the diff (git diff packages/llm-info/data/models.yml) and open a PR."
+ : "No changes — models.yml is up to date.",
+ );
+ } catch (error) {
+ Logger.error("Sync failed:", error);
+ process.exit(1);
+ }
+}
+
+if (
+ process.argv[1] &&
+ import.meta.url === pathToFileURL(process.argv[1]).href
+) {
+ main().catch((error) => {
+ Logger.error(error);
+ process.exit(1);
+ });
+}