Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(eve): filter model reasoning levels (#3919)

OwenKephartc94736abf7df
diff --git a/.changeset/filter-model-reasoning-levels.md b/.changeset/filter-model-reasoning-levels.md
new file mode 100644
index 000000000..a848c0678
--- /dev/null
+++ b/.changeset/filter-model-reasoning-levels.md
@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Filter `/model` reasoning choices to the selected model's supported effort levels from the AI Gateway catalog.
diff --git a/packages/eve/src/cli/dev/tui/tui.test.ts b/packages/eve/src/cli/dev/tui/tui.test.ts
index 989dda7d1..95df9698d 100644
--- a/packages/eve/src/cli/dev/tui/tui.test.ts
+++ b/packages/eve/src/cli/dev/tui/tui.test.ts
@@ -43,6 +43,45 @@ describe("runDevelopmentTui", () => {
expect(second.session).toBeUndefined();
});
+ it("offers default but no incompatible effort for toggle-only models", async () => {
+ vi.stubGlobal(
+ "fetch",
+ vi.fn().mockResolvedValue(
+ new Response(
+ JSON.stringify({
+ data: [
+ {
+ id: "anthropic/claude-toggle-only",
+ name: "Claude Toggle Only",
+ type: "language",
+ owned_by: "anthropic",
+ tags: ["reasoning", "web-search"],
+ reasoning_options: [{ type: "toggle" }],
+ },
+ ],
+ }),
+ ),
+ ),
+ );
+ await runDevelopmentTui({
+ target: {
+ kind: "local",
+ serverUrl: "http://127.0.0.1:4321/",
+ workspaceRoot: "/tmp/app",
+ },
+ });
+
+ const suggestions = await mocks.runnerOptions[0]?.argumentSuggestions?.("model");
+ expect(suggestions).toEqual([
+ {
+ value: "anthropic/claude-toggle-only",
+ label: "anthropic/claude-toggle-only",
+ hint: "Anthropic",
+ next: [{ value: "default", label: "default" }],
+ },
+ ]);
+ });
+
it.each([
[
"remote",
diff --git a/packages/eve/src/cli/dev/tui/tui.ts b/packages/eve/src/cli/dev/tui/tui.ts
index dad2710ea..063f019e1 100644
--- a/packages/eve/src/cli/dev/tui/tui.ts
+++ b/packages/eve/src/cli/dev/tui/tui.ts
@@ -77,19 +77,19 @@ function inlineArgumentSuggestions(appRoot: string) {
await import("#setup/boxes/select-model.js");
const catalog = await fetchGatewayCatalog().catch(() => undefined);
return modelOptionsFromCatalog(catalog).map((option) => {
- const reasoning = gatewayModelCapabilities(catalog, option.value)?.reasoningLevels;
+ const capabilities = gatewayModelCapabilities(catalog, option.value);
return {
value: option.value,
label: option.value,
hint: option.hint,
- ...(reasoning === undefined || reasoning.length === 0
- ? {}
- : {
+ ...(capabilities?.reasoning
+ ? {
next: [
{ value: "default", label: "default" },
- ...reasoning.map((value) => ({ value, label: value })),
+ ...capabilities.reasoningLevels.map((value) => ({ value, label: value })),
],
- }),
+ }
+ : {}),
};
});
}
diff --git a/packages/eve/src/setup/boxes/model-capabilities.test.ts b/packages/eve/src/setup/boxes/model-capabilities.test.ts
index 2e18b9ecf..837ef5adc 100644
--- a/packages/eve/src/setup/boxes/model-capabilities.test.ts
+++ b/packages/eve/src/setup/boxes/model-capabilities.test.ts
@@ -8,6 +8,7 @@ function entry(overrides: Partial<GatewayCatalogModel> & { id: string }): Gatewa
name: overrides.id,
type: "language",
owned_by: overrides.id.split("/")[0] ?? "",
+ reasoningEfforts: [],
...overrides,
};
}
@@ -16,18 +17,41 @@ const CATALOG: GatewayCatalogModel[] = [
entry({
id: "openai/gpt-5.5",
tags: ["reasoning", "web-search"],
+ reasoningEfforts: ["minimal", "high"],
pricing: { service_tiers: { priority: {}, flex: {} } },
}),
- entry({ id: "xai/grok-4.5", tags: ["reasoning", "web-search"] }),
- entry({ id: "deepseek/deepseek-v4-pro", tags: ["reasoning"] }),
- entry({ id: "xai/grok-4.20-non-reasoning", tags: ["web-search"], pricing: {} }),
+ entry({
+ id: "xai/grok-4.5",
+ tags: ["reasoning", "web-search"],
+ reasoningEfforts: ["low"],
+ }),
+ entry({
+ id: "deepseek/deepseek-v4-pro",
+ tags: ["reasoning"],
+ reasoningEfforts: ["none", "high", "max"],
+ }),
+ entry({
+ id: "xai/grok-4.20-non-reasoning",
+ tags: ["web-search"],
+ pricing: {},
+ reasoningEfforts: [],
+ }),
+ entry({
+ id: "anthropic/claude-toggle-only",
+ tags: ["reasoning"],
+ reasoningEfforts: [],
+ }),
+ entry({
+ id: "vendor/catalog-efforts-only",
+ reasoningEfforts: ["high"],
+ }),
];
describe("gatewayModelCapabilities", () => {
it("reads reasoning from the catalog tag and Fast mode from the priority tier", () => {
expect(gatewayModelCapabilities(CATALOG, "openai/gpt-5.5")).toEqual({
reasoning: true,
- reasoningLevels: ["none", "minimal", "low", "medium", "high", "xhigh"],
+ reasoningLevels: ["minimal", "high"],
fastMode: true,
});
expect(gatewayModelCapabilities(CATALOG, "xai/grok-4.20-non-reasoning")).toEqual({
@@ -35,19 +59,21 @@ describe("gatewayModelCapabilities", () => {
reasoningLevels: [],
fastMode: false,
});
+ expect(
+ gatewayModelCapabilities(CATALOG, "anthropic/claude-toggle-only")?.reasoningLevels,
+ ).toEqual([]);
});
- it("narrows reasoning levels per provider and falls back to the common core", () => {
- expect(gatewayModelCapabilities(CATALOG, "xai/grok-4.5")?.reasoningLevels).toEqual([
- "low",
- "high",
- ]);
+ it("uses only each model's supported reasoning levels", () => {
+ expect(gatewayModelCapabilities(CATALOG, "xai/grok-4.5")?.reasoningLevels).toEqual(["low"]);
expect(gatewayModelCapabilities(CATALOG, "deepseek/deepseek-v4-pro")?.reasoningLevels).toEqual([
"none",
- "low",
- "medium",
"high",
]);
+ expect(gatewayModelCapabilities(CATALOG, "vendor/catalog-efforts-only")).toMatchObject({
+ reasoning: true,
+ reasoningLevels: ["high"],
+ });
});
it("returns undefined without a catalog, a model id, or a catalog entry", () => {
diff --git a/packages/eve/src/setup/boxes/model-capabilities.ts b/packages/eve/src/setup/boxes/model-capabilities.ts
index 79706bffa..d2e363531 100644
--- a/packages/eve/src/setup/boxes/model-capabilities.ts
+++ b/packages/eve/src/setup/boxes/model-capabilities.ts
@@ -10,7 +10,7 @@ export type ReasoningLevel = Exclude<AgentReasoningDefinition, "provider-default
* `/model` menu paints so unsupported controls never present as available.
*/
export interface GatewayModelCapabilities {
- /** Whether the model advertises adjustable reasoning (the `reasoning` catalog tag). */
+ /** Whether the model advertises adjustable reasoning. */
readonly reasoning: boolean;
/** Effort levels worth offering for the model; empty when reasoning is unsupported. */
readonly reasoningLevels: readonly ReasoningLevel[];
@@ -18,7 +18,7 @@ export interface GatewayModelCapabilities {
readonly fastMode: boolean;
}
-/** Every effort level eve can author, offered when capabilities are unknown. */
+/** Every effort level eve can author. */
export const ALL_REASONING_LEVELS: readonly ReasoningLevel[] = [
"none",
"minimal",
@@ -28,17 +28,11 @@ export const ALL_REASONING_LEVELS: readonly ReasoningLevel[] = [
"xhigh",
];
-// The catalog carries no per-model level list, so levels are a best-effort
-// read of what each provider's API accepts today; unknown reasoning-capable
-// providers get the common core every provider maps.
-const REASONING_LEVELS_BY_PROVIDER: Record<string, readonly ReasoningLevel[]> = {
- anthropic: ALL_REASONING_LEVELS,
- openai: ALL_REASONING_LEVELS,
- google: ["none", "low", "medium", "high"],
- xai: ["low", "high"],
-};
-
-const DEFAULT_REASONING_LEVELS: readonly ReasoningLevel[] = ["none", "low", "medium", "high"];
+function reasoningLevels(model: GatewayCatalogModel): readonly ReasoningLevel[] {
+ return model.reasoningEfforts.filter((value): value is ReasoningLevel =>
+ ALL_REASONING_LEVELS.includes(value as ReasoningLevel),
+ );
+}
/**
* Capabilities for `modelId` from a fetched catalog, or undefined when the
@@ -53,14 +47,12 @@ export function gatewayModelCapabilities(
const model = catalog.find((entry) => entry.id === modelId);
if (model === undefined) return undefined;
- const reasoning = (model.tags ?? []).includes("reasoning");
- const provider = modelId.split("/")[0] ?? "";
+ const levels = reasoningLevels(model);
+ const reasoning = (model.tags ?? []).includes("reasoning") || levels.length > 0;
const tiers = model.pricing?.service_tiers;
return {
reasoning,
- reasoningLevels: reasoning
- ? (REASONING_LEVELS_BY_PROVIDER[provider] ?? DEFAULT_REASONING_LEVELS)
- : [],
+ reasoningLevels: levels,
fastMode: tiers !== undefined && Object.hasOwn(tiers, "priority"),
};
}
diff --git a/packages/eve/src/setup/boxes/select-model.test.ts b/packages/eve/src/setup/boxes/select-model.test.ts
index c82e0e56b..55d3338e2 100644
--- a/packages/eve/src/setup/boxes/select-model.test.ts
+++ b/packages/eve/src/setup/boxes/select-model.test.ts
@@ -35,6 +35,7 @@ const CATALOG: GatewayCatalogModel[] = [
owned_by: "zai",
released: 300,
tags: ["web-search"],
+ reasoningEfforts: [],
},
{
id: "openai/gpt-5-mini",
@@ -43,6 +44,7 @@ const CATALOG: GatewayCatalogModel[] = [
owned_by: "openai",
released: 200,
tags: ["web-search"],
+ reasoningEfforts: [],
},
{
id: "spacexai/grok-4.7",
@@ -51,11 +53,25 @@ const CATALOG: GatewayCatalogModel[] = [
owned_by: "spacexai",
released: 100,
tags: ["reasoning"],
+ reasoningEfforts: [],
},
// Filtered out: not a language model.
- { id: "openai/dall-e-3", name: "DALL-E 3", type: "image", owned_by: "openai" },
+ {
+ id: "openai/dall-e-3",
+ name: "DALL-E 3",
+ type: "image",
+ owned_by: "openai",
+ reasoningEfforts: [],
+ },
// Filtered out: missing the web-search tag.
- { id: "google/gemma-2", name: "Gemma 2", type: "language", owned_by: "google", tags: [] },
+ {
+ id: "google/gemma-2",
+ name: "Gemma 2",
+ type: "language",
+ owned_by: "google",
+ tags: [],
+ reasoningEfforts: [],
+ },
];
describe("modelOptionsFromCatalog", () => {
@@ -91,6 +107,7 @@ describe("modelOptionsFromCatalog", () => {
owned_by: "anthropic",
released: 400,
tags: ["web-search"],
+ reasoningEfforts: [],
},
...CATALOG,
]);
@@ -115,6 +132,7 @@ describe("modelOptionsFromCatalog", () => {
type: "language",
owned_by: "acme",
tags: ["web-search"],
+ reasoningEfforts: [],
};
if (released !== undefined) entry.released = released;
return entry;
@@ -150,7 +168,7 @@ describe("parseGatewayModelCatalog", () => {
expect(models.map((model) => model.id)).toEqual(["zai/glm-4.6", "openai/gpt-5-mini"]);
});
- it("keeps a model when only its optional catalog metadata is malformed", () => {
+ it("keeps valid reasoning efforts while ignoring malformed optional metadata", () => {
expect(
parseGatewayModelCatalog({
data: [
@@ -161,6 +179,11 @@ describe("parseGatewayModelCatalog", () => {
owned_by: "vendor",
released: "unannounced",
tags: null,
+ reasoning_options: [
+ { type: "toggle" },
+ { type: "effort", values: ["low", "high"] },
+ { type: "effort", values: null },
+ ],
},
],
}),
@@ -170,6 +193,7 @@ describe("parseGatewayModelCatalog", () => {
name: "Experimental",
type: "language",
owned_by: "vendor",
+ reasoningEfforts: ["low", "high"],
},
]);
});
diff --git a/packages/eve/src/shared/gateway-model-catalog.ts b/packages/eve/src/shared/gateway-model-catalog.ts
index fb76fe364..cd5195891 100644
--- a/packages/eve/src/shared/gateway-model-catalog.ts
+++ b/packages/eve/src/shared/gateway-model-catalog.ts
@@ -1,18 +1,33 @@
import { z } from "#compiled/zod/index.js";
-const gatewayCatalogModelSchema = z.object({
- id: z.string(),
- name: z.string(),
- type: z.string(),
- owned_by: z.string(),
- released: z.number().finite().optional().catch(undefined),
- tags: z.array(z.string()).optional().catch(undefined),
- pricing: z
- .object({ service_tiers: z.record(z.string(), z.unknown()).optional().catch(undefined) })
- .optional()
- .catch(undefined),
+const gatewayReasoningEffortSchema = z.object({
+ type: z.literal("effort"),
+ values: z.array(z.string()).catch([]),
});
+const gatewayCatalogModelSchema = z
+ .object({
+ id: z.string(),
+ name: z.string(),
+ type: z.string(),
+ owned_by: z.string(),
+ released: z.number().finite().optional().catch(undefined),
+ tags: z.array(z.string()).optional().catch(undefined),
+ reasoning_options: z.array(z.unknown()).optional().catch(undefined),
+ pricing: z
+ .object({ service_tiers: z.record(z.string(), z.unknown()).optional().catch(undefined) })
+ .optional()
+ .catch(undefined),
+ })
+ .transform(({ reasoning_options, ...model }) => ({
+ ...model,
+ reasoningEfforts:
+ reasoning_options?.flatMap((option) => {
+ const result = gatewayReasoningEffortSchema.safeParse(option);
+ return result.success ? result.data.values : [];
+ }) ?? [],
+ }));
+
const gatewayCatalogSchema = z.object({ data: z.array(z.unknown()) }).transform(({ data }) =>
data.flatMap((entry) => {
const result = gatewayCatalogModelSchema.safeParse(entry);