The Open Framework for Building Agents
fix(eve): filter model reasoning levels (#3919)
OwenKephartc94736abf7df
diff --git a/.changeset/filter-model-reasoning-levels.md b/.changeset/filter-model-reasoning-levels.mdnew file mode 100644index 000000000..a848c0678--- /dev/null+++ b/.changeset/filter-model-reasoning-levels.md@@ -0,0 +1,5 @@+---+"eve": patch+---++Filter `/model` reasoning choices to the selected model's supported effort levels from the AI Gateway catalog.diff --git a/packages/eve/src/cli/dev/tui/tui.test.ts b/packages/eve/src/cli/dev/tui/tui.test.tsindex 989dda7d1..95df9698d 100644--- a/packages/eve/src/cli/dev/tui/tui.test.ts+++ b/packages/eve/src/cli/dev/tui/tui.test.ts@@ -43,6 +43,45 @@ describe("runDevelopmentTui", () => {expect(second.session).toBeUndefined();});+ it("offers default but no incompatible effort for toggle-only models", async () => {+ vi.stubGlobal(+ "fetch",+ vi.fn().mockResolvedValue(+ new Response(+ JSON.stringify({+ data: [+ {+ id: "anthropic/claude-toggle-only",+ name: "Claude Toggle Only",+ type: "language",+ owned_by: "anthropic",+ tags: ["reasoning", "web-search"],+ reasoning_options: [{ type: "toggle" }],+ },+ ],+ }),+ ),+ ),+ );+ await runDevelopmentTui({+ target: {+ kind: "local",+ serverUrl: "http://127.0.0.1:4321/",+ workspaceRoot: "/tmp/app",+ },+ });++ const suggestions = await mocks.runnerOptions[0]?.argumentSuggestions?.("model");+ expect(suggestions).toEqual([+ {+ value: "anthropic/claude-toggle-only",+ label: "anthropic/claude-toggle-only",+ hint: "Anthropic",+ next: [{ value: "default", label: "default" }],+ },+ ]);+ });+it.each([["remote",diff --git a/packages/eve/src/cli/dev/tui/tui.ts b/packages/eve/src/cli/dev/tui/tui.tsindex dad2710ea..063f019e1 100644--- a/packages/eve/src/cli/dev/tui/tui.ts+++ b/packages/eve/src/cli/dev/tui/tui.ts@@ -77,19 +77,19 @@ function inlineArgumentSuggestions(appRoot: string) {await import("#setup/boxes/select-model.js");const catalog = await fetchGatewayCatalog().catch(() => undefined);return modelOptionsFromCatalog(catalog).map((option) => {- const reasoning = gatewayModelCapabilities(catalog, option.value)?.reasoningLevels;+ const capabilities = gatewayModelCapabilities(catalog, option.value);return {value: option.value,label: option.value,hint: option.hint,- ...(reasoning === undefined || reasoning.length === 0- ? {}- : {+ ...(capabilities?.reasoning+ ? {next: [{ value: "default", label: "default" },- ...reasoning.map((value) => ({ value, label: value })),+ ...capabilities.reasoningLevels.map((value) => ({ value, label: value })),],- }),+ }+ : {}),};});}diff --git a/packages/eve/src/setup/boxes/model-capabilities.test.ts b/packages/eve/src/setup/boxes/model-capabilities.test.tsindex 2e18b9ecf..837ef5adc 100644--- a/packages/eve/src/setup/boxes/model-capabilities.test.ts+++ b/packages/eve/src/setup/boxes/model-capabilities.test.ts@@ -8,6 +8,7 @@ function entry(overrides: Partial<GatewayCatalogModel> & { id: string }): Gatewaname: overrides.id,type: "language",owned_by: overrides.id.split("/")[0] ?? "",+ reasoningEfforts: [],...overrides,};}@@ -16,18 +17,41 @@ const CATALOG: GatewayCatalogModel[] = [entry({id: "openai/gpt-5.5",tags: ["reasoning", "web-search"],+ reasoningEfforts: ["minimal", "high"],pricing: { service_tiers: { priority: {}, flex: {} } },}),- entry({ id: "xai/grok-4.5", tags: ["reasoning", "web-search"] }),- entry({ id: "deepseek/deepseek-v4-pro", tags: ["reasoning"] }),- entry({ id: "xai/grok-4.20-non-reasoning", tags: ["web-search"], pricing: {} }),+ entry({+ id: "xai/grok-4.5",+ tags: ["reasoning", "web-search"],+ reasoningEfforts: ["low"],+ }),+ entry({+ id: "deepseek/deepseek-v4-pro",+ tags: ["reasoning"],+ reasoningEfforts: ["none", "high", "max"],+ }),+ entry({+ id: "xai/grok-4.20-non-reasoning",+ tags: ["web-search"],+ pricing: {},+ reasoningEfforts: [],+ }),+ entry({+ id: "anthropic/claude-toggle-only",+ tags: ["reasoning"],+ reasoningEfforts: [],+ }),+ entry({+ id: "vendor/catalog-efforts-only",+ reasoningEfforts: ["high"],+ }),];describe("gatewayModelCapabilities", () => {it("reads reasoning from the catalog tag and Fast mode from the priority tier", () => {expect(gatewayModelCapabilities(CATALOG, "openai/gpt-5.5")).toEqual({reasoning: true,- reasoningLevels: ["none", "minimal", "low", "medium", "high", "xhigh"],+ reasoningLevels: ["minimal", "high"],fastMode: true,});expect(gatewayModelCapabilities(CATALOG, "xai/grok-4.20-non-reasoning")).toEqual({@@ -35,19 +59,21 @@ describe("gatewayModelCapabilities", () => {reasoningLevels: [],fastMode: false,});+ expect(+ gatewayModelCapabilities(CATALOG, "anthropic/claude-toggle-only")?.reasoningLevels,+ ).toEqual([]);});- it("narrows reasoning levels per provider and falls back to the common core", () => {- expect(gatewayModelCapabilities(CATALOG, "xai/grok-4.5")?.reasoningLevels).toEqual([- "low",- "high",- ]);+ it("uses only each model's supported reasoning levels", () => {+ expect(gatewayModelCapabilities(CATALOG, "xai/grok-4.5")?.reasoningLevels).toEqual(["low"]);expect(gatewayModelCapabilities(CATALOG, "deepseek/deepseek-v4-pro")?.reasoningLevels).toEqual(["none",- "low",- "medium","high",]);+ expect(gatewayModelCapabilities(CATALOG, "vendor/catalog-efforts-only")).toMatchObject({+ reasoning: true,+ reasoningLevels: ["high"],+ });});it("returns undefined without a catalog, a model id, or a catalog entry", () => {diff --git a/packages/eve/src/setup/boxes/model-capabilities.ts b/packages/eve/src/setup/boxes/model-capabilities.tsindex 79706bffa..d2e363531 100644--- a/packages/eve/src/setup/boxes/model-capabilities.ts+++ b/packages/eve/src/setup/boxes/model-capabilities.ts@@ -10,7 +10,7 @@ export type ReasoningLevel = Exclude<AgentReasoningDefinition, "provider-default* `/model` menu paints so unsupported controls never present as available.*/export interface GatewayModelCapabilities {- /** Whether the model advertises adjustable reasoning (the `reasoning` catalog tag). */+ /** Whether the model advertises adjustable reasoning. */readonly reasoning: boolean;/** Effort levels worth offering for the model; empty when reasoning is unsupported. */readonly reasoningLevels: readonly ReasoningLevel[];@@ -18,7 +18,7 @@ export interface GatewayModelCapabilities {readonly fastMode: boolean;}-/** Every effort level eve can author, offered when capabilities are unknown. */+/** Every effort level eve can author. */export const ALL_REASONING_LEVELS: readonly ReasoningLevel[] = ["none","minimal",@@ -28,17 +28,11 @@ export const ALL_REASONING_LEVELS: readonly ReasoningLevel[] = ["xhigh",];-// The catalog carries no per-model level list, so levels are a best-effort-// read of what each provider's API accepts today; unknown reasoning-capable-// providers get the common core every provider maps.-const REASONING_LEVELS_BY_PROVIDER: Record<string, readonly ReasoningLevel[]> = {- anthropic: ALL_REASONING_LEVELS,- openai: ALL_REASONING_LEVELS,- google: ["none", "low", "medium", "high"],- xai: ["low", "high"],-};--const DEFAULT_REASONING_LEVELS: readonly ReasoningLevel[] = ["none", "low", "medium", "high"];+function reasoningLevels(model: GatewayCatalogModel): readonly ReasoningLevel[] {+ return model.reasoningEfforts.filter((value): value is ReasoningLevel =>+ ALL_REASONING_LEVELS.includes(value as ReasoningLevel),+ );+}/*** Capabilities for `modelId` from a fetched catalog, or undefined when the@@ -53,14 +47,12 @@ export function gatewayModelCapabilities(const model = catalog.find((entry) => entry.id === modelId);if (model === undefined) return undefined;- const reasoning = (model.tags ?? []).includes("reasoning");- const provider = modelId.split("/")[0] ?? "";+ const levels = reasoningLevels(model);+ const reasoning = (model.tags ?? []).includes("reasoning") || levels.length > 0;const tiers = model.pricing?.service_tiers;return {reasoning,- reasoningLevels: reasoning- ? (REASONING_LEVELS_BY_PROVIDER[provider] ?? DEFAULT_REASONING_LEVELS)- : [],+ reasoningLevels: levels,fastMode: tiers !== undefined && Object.hasOwn(tiers, "priority"),};}diff --git a/packages/eve/src/setup/boxes/select-model.test.ts b/packages/eve/src/setup/boxes/select-model.test.tsindex c82e0e56b..55d3338e2 100644--- a/packages/eve/src/setup/boxes/select-model.test.ts+++ b/packages/eve/src/setup/boxes/select-model.test.ts@@ -35,6 +35,7 @@ const CATALOG: GatewayCatalogModel[] = [owned_by: "zai",released: 300,tags: ["web-search"],+ reasoningEfforts: [],},{id: "openai/gpt-5-mini",@@ -43,6 +44,7 @@ const CATALOG: GatewayCatalogModel[] = [owned_by: "openai",released: 200,tags: ["web-search"],+ reasoningEfforts: [],},{id: "spacexai/grok-4.7",@@ -51,11 +53,25 @@ const CATALOG: GatewayCatalogModel[] = [owned_by: "spacexai",released: 100,tags: ["reasoning"],+ reasoningEfforts: [],},// Filtered out: not a language model.- { id: "openai/dall-e-3", name: "DALL-E 3", type: "image", owned_by: "openai" },+ {+ id: "openai/dall-e-3",+ name: "DALL-E 3",+ type: "image",+ owned_by: "openai",+ reasoningEfforts: [],+ },// Filtered out: missing the web-search tag.- { id: "google/gemma-2", name: "Gemma 2", type: "language", owned_by: "google", tags: [] },+ {+ id: "google/gemma-2",+ name: "Gemma 2",+ type: "language",+ owned_by: "google",+ tags: [],+ reasoningEfforts: [],+ },];describe("modelOptionsFromCatalog", () => {@@ -91,6 +107,7 @@ describe("modelOptionsFromCatalog", () => {owned_by: "anthropic",released: 400,tags: ["web-search"],+ reasoningEfforts: [],},...CATALOG,]);@@ -115,6 +132,7 @@ describe("modelOptionsFromCatalog", () => {type: "language",owned_by: "acme",tags: ["web-search"],+ reasoningEfforts: [],};if (released !== undefined) entry.released = released;return entry;@@ -150,7 +168,7 @@ describe("parseGatewayModelCatalog", () => {expect(models.map((model) => model.id)).toEqual(["zai/glm-4.6", "openai/gpt-5-mini"]);});- it("keeps a model when only its optional catalog metadata is malformed", () => {+ it("keeps valid reasoning efforts while ignoring malformed optional metadata", () => {expect(parseGatewayModelCatalog({data: [@@ -161,6 +179,11 @@ describe("parseGatewayModelCatalog", () => {owned_by: "vendor",released: "unannounced",tags: null,+ reasoning_options: [+ { type: "toggle" },+ { type: "effort", values: ["low", "high"] },+ { type: "effort", values: null },+ ],},],}),@@ -170,6 +193,7 @@ describe("parseGatewayModelCatalog", () => {name: "Experimental",type: "language",owned_by: "vendor",+ reasoningEfforts: ["low", "high"],},]);});diff --git a/packages/eve/src/shared/gateway-model-catalog.ts b/packages/eve/src/shared/gateway-model-catalog.tsindex fb76fe364..cd5195891 100644--- a/packages/eve/src/shared/gateway-model-catalog.ts+++ b/packages/eve/src/shared/gateway-model-catalog.ts@@ -1,18 +1,33 @@import { z } from "#compiled/zod/index.js";-const gatewayCatalogModelSchema = z.object({- id: z.string(),- name: z.string(),- type: z.string(),- owned_by: z.string(),- released: z.number().finite().optional().catch(undefined),- tags: z.array(z.string()).optional().catch(undefined),- pricing: z- .object({ service_tiers: z.record(z.string(), z.unknown()).optional().catch(undefined) })- .optional()- .catch(undefined),+const gatewayReasoningEffortSchema = z.object({+ type: z.literal("effort"),+ values: z.array(z.string()).catch([]),});+const gatewayCatalogModelSchema = z+ .object({+ id: z.string(),+ name: z.string(),+ type: z.string(),+ owned_by: z.string(),+ released: z.number().finite().optional().catch(undefined),+ tags: z.array(z.string()).optional().catch(undefined),+ reasoning_options: z.array(z.unknown()).optional().catch(undefined),+ pricing: z+ .object({ service_tiers: z.record(z.string(), z.unknown()).optional().catch(undefined) })+ .optional()+ .catch(undefined),+ })+ .transform(({ reasoning_options, ...model }) => ({+ ...model,+ reasoningEfforts:+ reasoning_options?.flatMap((option) => {+ const result = gatewayReasoningEffortSchema.safeParse(option);+ return result.success ? result.data.values : [];+ }) ?? [],+ }));+const gatewayCatalogSchema = z.object({ data: z.array(z.unknown()) }).transform(({ data }) =>data.flatMap((entry) => {const result = gatewayCatalogModelSchema.safeParse(entry);