The Open Framework for Building Agents
fix(eve): evaluate - reuse TUI Gateway authentication (#3452)
Andrew Barba40f0c4553923
diff --git a/.changeset/fuzzy-taxis-evaluate.md b/.changeset/fuzzy-taxis-evaluate.mdnew file mode 100644index 000000000..f79983482--- /dev/null+++ b/.changeset/fuzzy-taxis-evaluate.md@@ -0,0 +1,6 @@+---+"eve": patch+---++Fix `autoModel` to use `typesafe-ai/jev` and the `/login` Gateway connection during `eve dev`, while preserving custom AI SDK default providers.+Show `dynamic model` in the TUI footer instead of a missing-connection warning for dynamic selectors.diff --git a/docs/guides/evaluate.md b/docs/guides/evaluate.mdindex e64459e6c..970eb43a2 100644--- a/docs/guides/evaluate.md+++ b/docs/guides/evaluate.md@@ -14,7 +14,7 @@ releases.## Choose from Gateway models-By default, `autoModel` evaluates with `typesafe-ai/jev-latest`. Like other AI SDK+By default, `autoModel` evaluates with `typesafe-ai/jev`. Like other AI SDKmodel strings, it uses Vercel AI Gateway unless the application has configured adifferent global default provider.@@ -33,7 +33,11 @@ export default defineAgent({```Configure Gateway authentication as you would for any other AI SDK model. eve-does not add a TypeSafe credential or transport layer.+does not add a TypeSafe credential or transport layer. During `eve dev`, a+Gateway evaluator uses the same connection selected through `/login` as Gateway+language models. A configured AI SDK default provider still owns string model+resolution during development. The TUI footer displays `dynamic model` when the+agent uses `autoModel`.## Use a provider directlydiff --git a/packages/eve/src/cli/dev/tui/setup-issues.test.ts b/packages/eve/src/cli/dev/tui/setup-issues.test.tsindex 8d0127242..44638c1ae 100644--- a/packages/eve/src/cli/dev/tui/setup-issues.test.ts+++ b/packages/eve/src/cli/dev/tui/setup-issues.test.ts@@ -103,6 +103,25 @@ describe("BOOT_DETECTIONS", () => {expect(await detectSetupIssues(context({ info }))).toEqual([]);});+ it.each([{}, { EVE_MODEL_CONNECTION: "vercel" }, { AI_GATEWAY_API_KEY: "key" }])(+ "does not diagnose dynamic routing as a missing connection",+ async (env) => {+ const info = infoWithRouting({+ kind: "dynamic",+ resolver: {+ eventNames: ["step.started"],+ slug: "model",+ logicalPath: "agent.ts",+ owner: { kind: "application" },+ sourceId: "agent-model",+ sourceKind: "module",+ },+ });+ expect(await detectSetupIssues(context({ env, info }))).toEqual([]);+ expect(normalizeLocalModelEndpoint(info, env)).toBe(info);+ },+ );+it("stays quiet when the runtime resolved linked-project OIDC", async () => {const info = infoWithRouting({ kind: "gateway", target: "openai" },diff --git a/packages/eve/src/cli/dev/tui/setup-issues.ts b/packages/eve/src/cli/dev/tui/setup-issues.tsindex 6694067fd..4f280a693 100644--- a/packages/eve/src/cli/dev/tui/setup-issues.ts+++ b/packages/eve/src/cli/dev/tui/setup-issues.ts@@ -36,6 +36,7 @@ export interface BootDetection {type ModelProviderAccess =| { kind: "unknown" }+ | { kind: "dynamic" }| { kind: "external" }| {kind: "gateway";@@ -95,6 +96,7 @@ function modelProviderAccess(context: Pick<BootDetectionContext, "env" | "info">,): ModelProviderAccess {const model = context.info?.agent.model;+ if (model?.routing?.kind === "dynamic") return { kind: "dynamic" };if (model?.routing?.kind === "external") return { kind: "external" };if (model?.routing?.kind !== "gateway") return { kind: "unknown" };@@ -144,7 +146,9 @@ const modelProvider: BootDetection = {async detect({ appRoot, env, info }) {const access = modelProviderAccess({ env, info });- if (access.kind === "external") return [];+ // Dynamic selectors can return any provider; their credentials cannot be+ // diagnosed until a model is selected at runtime.+ if (access.kind === "external" || access.kind === "dynamic") return [];if (access.kind === "gateway") {if (access.runtime.status === "connected") return [];if (access.runtime.status === "disconnected") {diff --git a/packages/eve/src/cli/dev/tui/terminal-renderer.test.ts b/packages/eve/src/cli/dev/tui/terminal-renderer.test.tsindex a367bff02..828d488ff 100644--- a/packages/eve/src/cli/dev/tui/terminal-renderer.test.ts+++ b/packages/eve/src/cli/dev/tui/terminal-renderer.test.ts@@ -5350,6 +5350,47 @@ describe("TerminalRenderer status line", () => {renderer.shutdown();});+ it("shows dynamic model below the prompt after replacing a static model", async () => {+ const { screen, input, renderer } = makeRenderer();+ const info = agentInfoWithModel("openai/gpt-5.6-sol");+ const header = { name: "Weather Agent", serverUrl: "http://localhost:3000", info };+ renderer.renderAgentHeader(header);+ const prompt = renderer.readPrompt();+ renderer.renderAgentHeader({+ ...header,+ info: {+ ...info,+ agent: {+ ...info.agent,+ model: {+ routing: {+ kind: "dynamic",+ resolver: {+ eventNames: ["step.started"],+ slug: "model",+ logicalPath: "agent.ts",+ owner: { kind: "application" },+ sourceId: "agent-model",+ sourceKind: "module",+ },+ },+ },+ },+ },+ });+ const lines = screen.snapshot().split("\n");+ const promptRow = lines.findIndex((line) => line.includes("❯"));+ expect(promptRow).toBeGreaterThan(-1);+ const footer = lines.slice(promptRow + 1).join("\n");+ expect(footer).toContain("dynamic model");+ expect(footer).not.toContain("openai/gpt-5.6-sol");+ expect(footer).not.toContain("⚠ ai-gateway");+ input.type("done");+ input.enter();+ await prompt;+ renderer.shutdown();+ });+it("suppresses the status line while a setup flow panel is open", () => {const { screen, renderer } = makeRenderer();renderer.renderNotice("anchor");diff --git a/packages/eve/src/cli/dev/tui/terminal-renderer.ts b/packages/eve/src/cli/dev/tui/terminal-renderer.tsindex 69a35987e..2b9525d07 100644--- a/packages/eve/src/cli/dev/tui/terminal-renderer.ts+++ b/packages/eve/src/cli/dev/tui/terminal-renderer.ts@@ -4533,7 +4533,8 @@ export class TerminalRenderer implements AgentTUIRenderer {}if (this.#logLevelHintActive) input.logLevel = this.#logs;const agentModel = this.#agentHeader?.info?.agent.model;- if (agentModel?.id !== undefined) input.model = agentModel.id;+ if (agentModel?.routing.kind === "dynamic") input.model = "dynamic model";+ else if (agentModel?.id !== undefined) input.model = agentModel.id;// "provider-default" is the absent-setting sentinel, not a level worth showing.if (agentModel?.reasoning !== undefined && agentModel.reasoning !== "provider-default") {input.reasoning = agentModel.reasoning;diff --git a/packages/eve/src/experimental/evaluate/auto-model.integration.test.ts b/packages/eve/src/experimental/evaluate/auto-model.integration.test.tsindex 023ef840c..e2ffb6155 100644--- a/packages/eve/src/experimental/evaluate/auto-model.integration.test.ts+++ b/packages/eve/src/experimental/evaluate/auto-model.integration.test.ts@@ -8,11 +8,18 @@ import { anthropic } from "#public/models/anthropic/index.js";import { autoModel } from "./auto-model.js";-const runtime = vi.hoisted(() => ({ state: undefined as ContextContainer | undefined }));+const runtime = vi.hoisted(() => ({+ localEvaluationModel: vi.fn(),+ state: undefined as ContextContainer | undefined,+}));vi.mock("#context/container.js", async (importOriginal) => ({...(await importOriginal<typeof import("#context/container.js")>()),loadContext: () => runtime.state!,}));+vi.mock("#internal/model-auth/transport.js", async (importOriginal) => ({+ ...(await importOriginal<typeof import("#internal/model-auth/transport.js")>()),+ localGatewayEvaluationModel: runtime.localEvaluationModel,+}));const options = {"openai/large": "Difficult investigations",@@ -51,6 +58,7 @@ function evaluationModel(choice = "openai/small", modelId = "fixture-evaluator")beforeEach(() => {runtime.state = new ContextContainer();+ runtime.localEvaluationModel.mockReset();});describe("autoModel", () => {@@ -62,13 +70,45 @@ describe("autoModel", () => {try {const handler = autoModel({ options }).events["step.started"]!;await expect(handler(event(), context())).resolves.toBe("openai/small");- expect(evaluationModelFactory).toHaveBeenCalledWith("typesafe-ai/jev-latest");+ expect(runtime.localEvaluationModel).not.toHaveBeenCalled();+ expect(evaluationModelFactory).toHaveBeenCalledWith("typesafe-ai/jev");} finally {if (previous === undefined) Reflect.deleteProperty(globalThis, "AI_SDK_DEFAULT_PROVIDER");else Reflect.set(globalThis, "AI_SDK_DEFAULT_PROVIDER", previous);}});+ it("preserves a custom default provider when the local Gateway connection is available", async () => {+ const evaluator = evaluationModel();+ const localEvaluator = evaluationModel("openai/large");+ runtime.localEvaluationModel.mockReturnValue(localEvaluator.model);+ const evaluationModelFactory = vi.fn(() => evaluator.model);+ const previous = Reflect.get(globalThis, "AI_SDK_DEFAULT_PROVIDER");+ Reflect.set(globalThis, "AI_SDK_DEFAULT_PROVIDER", { evaluationModel: evaluationModelFactory });+ try {+ const handler = autoModel({ model: "internal-router", options }).events["step.started"]!;+ await expect(handler(event(), context())).resolves.toBe("openai/small");+ expect(evaluationModelFactory).toHaveBeenCalledWith("internal-router");+ expect(evaluator.doEvaluate).toHaveBeenCalledOnce();+ expect(runtime.localEvaluationModel).not.toHaveBeenCalled();+ expect(localEvaluator.doEvaluate).not.toHaveBeenCalled();+ } finally {+ if (previous === undefined) Reflect.deleteProperty(globalThis, "AI_SDK_DEFAULT_PROVIDER");+ else Reflect.set(globalThis, "AI_SDK_DEFAULT_PROVIDER", previous);+ }+ });++ it("uses the local Gateway connection for a string evaluation model", async () => {+ const evaluator = evaluationModel();+ runtime.localEvaluationModel.mockReturnValue(evaluator.model);++ const handler = autoModel({ options }).events["step.started"]!;++ await expect(handler(event(), context())).resolves.toBe("openai/small");+ expect(runtime.localEvaluationModel).toHaveBeenCalledWith("typesafe-ai/jev");+ expect(evaluator.doEvaluate).toHaveBeenCalledOnce();+ });+it("routes provider language models by alias and preserves reasoning", async () => {const languageModel = anthropic("sonnet-5");const evaluator = evaluationModel("private");diff --git a/packages/eve/src/experimental/evaluate/auto-model.ts b/packages/eve/src/experimental/evaluate/auto-model.tsindex 2fbe4bee4..56756fef3 100644--- a/packages/eve/src/experimental/evaluate/auto-model.ts+++ b/packages/eve/src/experimental/evaluate/auto-model.ts@@ -13,6 +13,7 @@ import {type DynamicSentinel,} from "#dynamic/definition.js";import { isAgentReasoningDefinition, isRuntimeLanguageModel } from "#internal/runtime-model.js";+import { localGatewayEvaluationModel } from "#internal/model-auth/transport.js";import type {AgentReasoningDefinition,PublicAgentDynamicModelResult,@@ -35,7 +36,7 @@ interface AutoModelConfig<readonly options: T;}-const DEFAULT_EVALUATION_MODEL = "typesafe-ai/jev-latest";+const DEFAULT_EVALUATION_MODEL = "typesafe-ai/jev";function isRecord(value: unknown): value is Record<string, unknown> {return typeof value === "object" && value !== null && !Array.isArray(value);@@ -168,7 +169,11 @@ export function autoModel<const T extends Readonly<Record<string, AutoModelOptioif (previous?.turnId === currentTurnId) return models.get(previous.model)!;const result = await evaluate({- model: evaluationModel,+ model:+ typeof evaluationModel === "string" &&+ Reflect.get(globalThis, "AI_SDK_DEFAULT_PROVIDER") == null+ ? (localGatewayEvaluationModel(evaluationModel) ?? evaluationModel)+ : evaluationModel,state: routingState(ctx),questions: {route: {diff --git a/packages/eve/src/internal/aisdk-provider-contract.test.ts b/packages/eve/src/internal/aisdk-provider-contract.test.tsindex a9bb98b12..f2eb6f06b 100644--- a/packages/eve/src/internal/aisdk-provider-contract.test.ts+++ b/packages/eve/src/internal/aisdk-provider-contract.test.ts@@ -25,7 +25,7 @@ describe("ai-sdk provider identity contract (canary)", () => {(globalThis as { AI_SDK_DEFAULT_PROVIDER?: unknown }).AI_SDK_DEFAULT_PROVIDER,).toBeUndefined();expect(gateway.languageModel("anthropic/claude-sonnet-5").provider).toBe("gateway");- expect(gateway.evaluationModel("typesafe-ai/jev-latest").provider).toBe("gateway");+ expect(gateway.evaluationModel("typesafe-ai/jev").provider).toBe("gateway");});it("direct provider instances do not report 'gateway'", () => {diff --git a/packages/eve/src/internal/model-auth/development-broker.integration.test.ts b/packages/eve/src/internal/model-auth/development-broker.integration.test.tsindex dfe5f6714..8d2f149c0 100644--- a/packages/eve/src/internal/model-auth/development-broker.integration.test.ts+++ b/packages/eve/src/internal/model-auth/development-broker.integration.test.ts@@ -23,7 +23,11 @@ vi.mock("#public/models/openai/chatgpt/token-broker.js", () => ({}));import { handleDevelopmentModelCredentialRequest } from "./development-broker-server.js";import { readDevelopmentModelCredential } from "./development-broker-client.js";-import { createDirectModelFetch, localGatewayModel } from "./transport.js";+import {+ createDirectModelFetch,+ localGatewayEvaluationModel,+ localGatewayModel,+} from "./transport.js";import { createCodexFetch } from "#public/models/openai/chatgpt/transport.js";const origin = "http://localhost:4567";@@ -220,6 +224,39 @@ it("streams through the same Gateway model after switching teams and then to a k}});+it("evaluates through the Gateway connection selected by /login", async () => {+ const model = localGatewayEvaluationModel("typesafe-ai/jev");+ if (!model) throw new Error("Expected a Gateway evaluation model");+ expect(JSON.stringify(model)).not.toContain("account-token");+ expect(mocks.gateway).not.toHaveBeenCalled();+ upstream.mockResolvedValue(+ Response.json({+ answers: { route: { type: "choice", choice: "fast" } },+ usage: { inputTokens: 4, outputTokens: 1 },+ }),+ );++ const result = await model.doEvaluate({+ state: "Alice requests a routine summary.",+ questions: {+ route: {+ type: "choice",+ instructions: "Choose a model.",+ criteria: { fast: "Routine work", thorough: "Difficult work" },+ },+ },+ });++ expect(result.answers.route).toEqual({ type: "choice", choice: "fast" });+ const [url, init] = upstream.mock.calls[0]!;+ expect(String(url)).toBe("https://ai-gateway.vercel.sh/v4/ai/evaluation-model");+ expect(init.headers.get("authorization")).toBe("Bearer account-token");+ expect(init.headers.get("x-vercel-ai-gateway-team")).toBe("team_a");+ expect(init.headers.get("ai-model-id")).toBe("typesafe-ai/jev");+ expect(init.headers.has(DEVELOPMENT_WORKFLOW_TRANSPORT_HEADER)).toBe(false);+ expect(init.body).not.toContain("account-token");+});+it("cancels credential lookup with the model request", async () => {const controller = new AbortController();controller.abort(new Error("request-cancelled"));diff --git a/packages/eve/src/internal/model-auth/transport.ts b/packages/eve/src/internal/model-auth/transport.tsindex 72c604b2a..19597a6ca 100644--- a/packages/eve/src/internal/model-auth/transport.ts+++ b/packages/eve/src/internal/model-auth/transport.ts@@ -1,4 +1,8 @@-import { createGateway, type LanguageModel } from "ai";+import {+ createGateway,+ type Experimental_EvaluationModel as EvaluationModel,+ type LanguageModel,+} from "ai";import { isEveDevEnvironment } from "#internal/application/dev-environment.js";import { resolveModelApiKey } from "./api-key.js";import {@@ -25,8 +29,7 @@ export function createDirectModelFetch(provider: "openai" | "anthropic"): typeof};}-/** Resolve the connection for every request, including models constructed before /login. */-export function localGatewayModel(id: string): LanguageModel | undefined {+function createLocalGateway(): ReturnType<typeof createGateway> | undefined {if (!isEveDevEnvironment()) return undefined;const selected = process.env[MODEL_CONNECTION_ENV];if (@@ -62,5 +65,17 @@ export function localGatewayModel(id: string): LanguageModel | undefined {authenticate(await resolve(credential.token, init?.signal));return fetch(url, { ...init, headers });},- })(id);+ });+}++/** Resolve the connection for every request, including models constructed before /login. */+export function localGatewayModel(id: string): LanguageModel | undefined {+ return createLocalGateway()?.languageModel(id);+}++/** Resolve an evaluation model through the same local Gateway connection used by language models. */+export function localGatewayEvaluationModel(+ id: string,+): Exclude<EvaluationModel, string> | undefined {+ return createLocalGateway()?.evaluationModel(id);}diff --git a/research/jev-decision-models.md b/research/jev-decision-models.mdindex ec630cde6..aa6d29ed0 100644--- a/research/jev-decision-models.md+++ b/research/jev-decision-models.md@@ -11,7 +11,7 @@ last_updated: "2026-09-16"Add only `autoModel` at `eve/experimental/evaluate`. Build it on AI SDK's`experimental_evaluate` API instead of owning a TypeSafe client, AI Gatewaytransport, credential resolver, or public decision API. Default to the-`typesafe-ai/jev-latest` model string. AI SDK model strings use its default+`typesafe-ai/jev` model string. AI SDK model strings use its defaultprovider, normally Vercel AI Gateway; explicit model instances use the providerpackage that created them.@@ -121,7 +121,8 @@ evaluation model adapters that use structured language-model output. Thoseadapters evaluate all questions in one prompt, do not reproduce Jev's nativeindependent-question execution, and omit Choice and Score distributions.-The AI SDK documentation uses `typesafe-ai/jev-latest` as the Gateway model ID.+The [Gateway model catalog](https://vercel.com/ai-gateway/models/jev) uses+`typesafe-ai/jev` as the Gateway model ID.Using it as an AI SDK string keeps Gateway API keys, Vercel OIDC, request headers,endpoint selection, and future protocol changes inside AI SDK. A direct TypeSafesetup installs `@ai-sdk/typesafe-ai` and supplies its model instance. eve does not