Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

fix(eve): evaluate - reuse TUI Gateway authentication (#3452)

Andrew Barba40f0c4553923
diff --git a/.changeset/fuzzy-taxis-evaluate.md b/.changeset/fuzzy-taxis-evaluate.md
new file mode 100644
index 000000000..f79983482
--- /dev/null
+++ b/.changeset/fuzzy-taxis-evaluate.md
@@ -0,0 +1,6 @@
+---
+"eve": patch
+---
+
+Fix `autoModel` to use `typesafe-ai/jev` and the `/login` Gateway connection during `eve dev`, while preserving custom AI SDK default providers.
+Show `dynamic model` in the TUI footer instead of a missing-connection warning for dynamic selectors.
diff --git a/docs/guides/evaluate.md b/docs/guides/evaluate.md
index e64459e6c..970eb43a2 100644
--- a/docs/guides/evaluate.md
+++ b/docs/guides/evaluate.md
@@ -14,7 +14,7 @@ releases.
## Choose from Gateway models
-By default, `autoModel` evaluates with `typesafe-ai/jev-latest`. Like other AI SDK
+By default, `autoModel` evaluates with `typesafe-ai/jev`. Like other AI SDK
model strings, it uses Vercel AI Gateway unless the application has configured a
different global default provider.
@@ -33,7 +33,11 @@ export default defineAgent({
```
Configure Gateway authentication as you would for any other AI SDK model. eve
-does not add a TypeSafe credential or transport layer.
+does not add a TypeSafe credential or transport layer. During `eve dev`, a
+Gateway evaluator uses the same connection selected through `/login` as Gateway
+language models. A configured AI SDK default provider still owns string model
+resolution during development. The TUI footer displays `dynamic model` when the
+agent uses `autoModel`.
## Use a provider directly
diff --git a/packages/eve/src/cli/dev/tui/setup-issues.test.ts b/packages/eve/src/cli/dev/tui/setup-issues.test.ts
index 8d0127242..44638c1ae 100644
--- a/packages/eve/src/cli/dev/tui/setup-issues.test.ts
+++ b/packages/eve/src/cli/dev/tui/setup-issues.test.ts
@@ -103,6 +103,25 @@ describe("BOOT_DETECTIONS", () => {
expect(await detectSetupIssues(context({ info }))).toEqual([]);
});
+ it.each([{}, { EVE_MODEL_CONNECTION: "vercel" }, { AI_GATEWAY_API_KEY: "key" }])(
+ "does not diagnose dynamic routing as a missing connection",
+ async (env) => {
+ const info = infoWithRouting({
+ kind: "dynamic",
+ resolver: {
+ eventNames: ["step.started"],
+ slug: "model",
+ logicalPath: "agent.ts",
+ owner: { kind: "application" },
+ sourceId: "agent-model",
+ sourceKind: "module",
+ },
+ });
+ expect(await detectSetupIssues(context({ env, info }))).toEqual([]);
+ expect(normalizeLocalModelEndpoint(info, env)).toBe(info);
+ },
+ );
+
it("stays quiet when the runtime resolved linked-project OIDC", async () => {
const info = infoWithRouting(
{ kind: "gateway", target: "openai" },
diff --git a/packages/eve/src/cli/dev/tui/setup-issues.ts b/packages/eve/src/cli/dev/tui/setup-issues.ts
index 6694067fd..4f280a693 100644
--- a/packages/eve/src/cli/dev/tui/setup-issues.ts
+++ b/packages/eve/src/cli/dev/tui/setup-issues.ts
@@ -36,6 +36,7 @@ export interface BootDetection {
type ModelProviderAccess =
| { kind: "unknown" }
+ | { kind: "dynamic" }
| { kind: "external" }
| {
kind: "gateway";
@@ -95,6 +96,7 @@ function modelProviderAccess(
context: Pick<BootDetectionContext, "env" | "info">,
): ModelProviderAccess {
const model = context.info?.agent.model;
+ if (model?.routing?.kind === "dynamic") return { kind: "dynamic" };
if (model?.routing?.kind === "external") return { kind: "external" };
if (model?.routing?.kind !== "gateway") return { kind: "unknown" };
@@ -144,7 +146,9 @@ const modelProvider: BootDetection = {
async detect({ appRoot, env, info }) {
const access = modelProviderAccess({ env, info });
- if (access.kind === "external") return [];
+ // Dynamic selectors can return any provider; their credentials cannot be
+ // diagnosed until a model is selected at runtime.
+ if (access.kind === "external" || access.kind === "dynamic") return [];
if (access.kind === "gateway") {
if (access.runtime.status === "connected") return [];
if (access.runtime.status === "disconnected") {
diff --git a/packages/eve/src/cli/dev/tui/terminal-renderer.test.ts b/packages/eve/src/cli/dev/tui/terminal-renderer.test.ts
index a367bff02..828d488ff 100644
--- a/packages/eve/src/cli/dev/tui/terminal-renderer.test.ts
+++ b/packages/eve/src/cli/dev/tui/terminal-renderer.test.ts
@@ -5350,6 +5350,47 @@ describe("TerminalRenderer status line", () => {
renderer.shutdown();
});
+ it("shows dynamic model below the prompt after replacing a static model", async () => {
+ const { screen, input, renderer } = makeRenderer();
+ const info = agentInfoWithModel("openai/gpt-5.6-sol");
+ const header = { name: "Weather Agent", serverUrl: "http://localhost:3000", info };
+ renderer.renderAgentHeader(header);
+ const prompt = renderer.readPrompt();
+ renderer.renderAgentHeader({
+ ...header,
+ info: {
+ ...info,
+ agent: {
+ ...info.agent,
+ model: {
+ routing: {
+ kind: "dynamic",
+ resolver: {
+ eventNames: ["step.started"],
+ slug: "model",
+ logicalPath: "agent.ts",
+ owner: { kind: "application" },
+ sourceId: "agent-model",
+ sourceKind: "module",
+ },
+ },
+ },
+ },
+ },
+ });
+ const lines = screen.snapshot().split("\n");
+ const promptRow = lines.findIndex((line) => line.includes("❯"));
+ expect(promptRow).toBeGreaterThan(-1);
+ const footer = lines.slice(promptRow + 1).join("\n");
+ expect(footer).toContain("dynamic model");
+ expect(footer).not.toContain("openai/gpt-5.6-sol");
+ expect(footer).not.toContain("⚠ ai-gateway");
+ input.type("done");
+ input.enter();
+ await prompt;
+ renderer.shutdown();
+ });
+
it("suppresses the status line while a setup flow panel is open", () => {
const { screen, renderer } = makeRenderer();
renderer.renderNotice("anchor");
diff --git a/packages/eve/src/cli/dev/tui/terminal-renderer.ts b/packages/eve/src/cli/dev/tui/terminal-renderer.ts
index 69a35987e..2b9525d07 100644
--- a/packages/eve/src/cli/dev/tui/terminal-renderer.ts
+++ b/packages/eve/src/cli/dev/tui/terminal-renderer.ts
@@ -4533,7 +4533,8 @@ export class TerminalRenderer implements AgentTUIRenderer {
}
if (this.#logLevelHintActive) input.logLevel = this.#logs;
const agentModel = this.#agentHeader?.info?.agent.model;
- if (agentModel?.id !== undefined) input.model = agentModel.id;
+ if (agentModel?.routing.kind === "dynamic") input.model = "dynamic model";
+ else if (agentModel?.id !== undefined) input.model = agentModel.id;
// "provider-default" is the absent-setting sentinel, not a level worth showing.
if (agentModel?.reasoning !== undefined && agentModel.reasoning !== "provider-default") {
input.reasoning = agentModel.reasoning;
diff --git a/packages/eve/src/experimental/evaluate/auto-model.integration.test.ts b/packages/eve/src/experimental/evaluate/auto-model.integration.test.ts
index 023ef840c..e2ffb6155 100644
--- a/packages/eve/src/experimental/evaluate/auto-model.integration.test.ts
+++ b/packages/eve/src/experimental/evaluate/auto-model.integration.test.ts
@@ -8,11 +8,18 @@ import { anthropic } from "#public/models/anthropic/index.js";
import { autoModel } from "./auto-model.js";
-const runtime = vi.hoisted(() => ({ state: undefined as ContextContainer | undefined }));
+const runtime = vi.hoisted(() => ({
+ localEvaluationModel: vi.fn(),
+ state: undefined as ContextContainer | undefined,
+}));
vi.mock("#context/container.js", async (importOriginal) => ({
...(await importOriginal<typeof import("#context/container.js")>()),
loadContext: () => runtime.state!,
}));
+vi.mock("#internal/model-auth/transport.js", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("#internal/model-auth/transport.js")>()),
+ localGatewayEvaluationModel: runtime.localEvaluationModel,
+}));
const options = {
"openai/large": "Difficult investigations",
@@ -51,6 +58,7 @@ function evaluationModel(choice = "openai/small", modelId = "fixture-evaluator")
beforeEach(() => {
runtime.state = new ContextContainer();
+ runtime.localEvaluationModel.mockReset();
});
describe("autoModel", () => {
@@ -62,13 +70,45 @@ describe("autoModel", () => {
try {
const handler = autoModel({ options }).events["step.started"]!;
await expect(handler(event(), context())).resolves.toBe("openai/small");
- expect(evaluationModelFactory).toHaveBeenCalledWith("typesafe-ai/jev-latest");
+ expect(runtime.localEvaluationModel).not.toHaveBeenCalled();
+ expect(evaluationModelFactory).toHaveBeenCalledWith("typesafe-ai/jev");
} finally {
if (previous === undefined) Reflect.deleteProperty(globalThis, "AI_SDK_DEFAULT_PROVIDER");
else Reflect.set(globalThis, "AI_SDK_DEFAULT_PROVIDER", previous);
}
});
+ it("preserves a custom default provider when the local Gateway connection is available", async () => {
+ const evaluator = evaluationModel();
+ const localEvaluator = evaluationModel("openai/large");
+ runtime.localEvaluationModel.mockReturnValue(localEvaluator.model);
+ const evaluationModelFactory = vi.fn(() => evaluator.model);
+ const previous = Reflect.get(globalThis, "AI_SDK_DEFAULT_PROVIDER");
+ Reflect.set(globalThis, "AI_SDK_DEFAULT_PROVIDER", { evaluationModel: evaluationModelFactory });
+ try {
+ const handler = autoModel({ model: "internal-router", options }).events["step.started"]!;
+ await expect(handler(event(), context())).resolves.toBe("openai/small");
+ expect(evaluationModelFactory).toHaveBeenCalledWith("internal-router");
+ expect(evaluator.doEvaluate).toHaveBeenCalledOnce();
+ expect(runtime.localEvaluationModel).not.toHaveBeenCalled();
+ expect(localEvaluator.doEvaluate).not.toHaveBeenCalled();
+ } finally {
+ if (previous === undefined) Reflect.deleteProperty(globalThis, "AI_SDK_DEFAULT_PROVIDER");
+ else Reflect.set(globalThis, "AI_SDK_DEFAULT_PROVIDER", previous);
+ }
+ });
+
+ it("uses the local Gateway connection for a string evaluation model", async () => {
+ const evaluator = evaluationModel();
+ runtime.localEvaluationModel.mockReturnValue(evaluator.model);
+
+ const handler = autoModel({ options }).events["step.started"]!;
+
+ await expect(handler(event(), context())).resolves.toBe("openai/small");
+ expect(runtime.localEvaluationModel).toHaveBeenCalledWith("typesafe-ai/jev");
+ expect(evaluator.doEvaluate).toHaveBeenCalledOnce();
+ });
+
it("routes provider language models by alias and preserves reasoning", async () => {
const languageModel = anthropic("sonnet-5");
const evaluator = evaluationModel("private");
diff --git a/packages/eve/src/experimental/evaluate/auto-model.ts b/packages/eve/src/experimental/evaluate/auto-model.ts
index 2fbe4bee4..56756fef3 100644
--- a/packages/eve/src/experimental/evaluate/auto-model.ts
+++ b/packages/eve/src/experimental/evaluate/auto-model.ts
@@ -13,6 +13,7 @@ import {
type DynamicSentinel,
} from "#dynamic/definition.js";
import { isAgentReasoningDefinition, isRuntimeLanguageModel } from "#internal/runtime-model.js";
+import { localGatewayEvaluationModel } from "#internal/model-auth/transport.js";
import type {
AgentReasoningDefinition,
PublicAgentDynamicModelResult,
@@ -35,7 +36,7 @@ interface AutoModelConfig<
readonly options: T;
}
-const DEFAULT_EVALUATION_MODEL = "typesafe-ai/jev-latest";
+const DEFAULT_EVALUATION_MODEL = "typesafe-ai/jev";
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
@@ -168,7 +169,11 @@ export function autoModel<const T extends Readonly<Record<string, AutoModelOptio
if (previous?.turnId === currentTurnId) return models.get(previous.model)!;
const result = await evaluate({
- model: evaluationModel,
+ model:
+ typeof evaluationModel === "string" &&
+ Reflect.get(globalThis, "AI_SDK_DEFAULT_PROVIDER") == null
+ ? (localGatewayEvaluationModel(evaluationModel) ?? evaluationModel)
+ : evaluationModel,
state: routingState(ctx),
questions: {
route: {
diff --git a/packages/eve/src/internal/aisdk-provider-contract.test.ts b/packages/eve/src/internal/aisdk-provider-contract.test.ts
index a9bb98b12..f2eb6f06b 100644
--- a/packages/eve/src/internal/aisdk-provider-contract.test.ts
+++ b/packages/eve/src/internal/aisdk-provider-contract.test.ts
@@ -25,7 +25,7 @@ describe("ai-sdk provider identity contract (canary)", () => {
(globalThis as { AI_SDK_DEFAULT_PROVIDER?: unknown }).AI_SDK_DEFAULT_PROVIDER,
).toBeUndefined();
expect(gateway.languageModel("anthropic/claude-sonnet-5").provider).toBe("gateway");
- expect(gateway.evaluationModel("typesafe-ai/jev-latest").provider).toBe("gateway");
+ expect(gateway.evaluationModel("typesafe-ai/jev").provider).toBe("gateway");
});
it("direct provider instances do not report 'gateway'", () => {
diff --git a/packages/eve/src/internal/model-auth/development-broker.integration.test.ts b/packages/eve/src/internal/model-auth/development-broker.integration.test.ts
index dfe5f6714..8d2f149c0 100644
--- a/packages/eve/src/internal/model-auth/development-broker.integration.test.ts
+++ b/packages/eve/src/internal/model-auth/development-broker.integration.test.ts
@@ -23,7 +23,11 @@ vi.mock("#public/models/openai/chatgpt/token-broker.js", () => ({
}));
import { handleDevelopmentModelCredentialRequest } from "./development-broker-server.js";
import { readDevelopmentModelCredential } from "./development-broker-client.js";
-import { createDirectModelFetch, localGatewayModel } from "./transport.js";
+import {
+ createDirectModelFetch,
+ localGatewayEvaluationModel,
+ localGatewayModel,
+} from "./transport.js";
import { createCodexFetch } from "#public/models/openai/chatgpt/transport.js";
const origin = "http://localhost:4567";
@@ -220,6 +224,39 @@ it("streams through the same Gateway model after switching teams and then to a k
}
});
+it("evaluates through the Gateway connection selected by /login", async () => {
+ const model = localGatewayEvaluationModel("typesafe-ai/jev");
+ if (!model) throw new Error("Expected a Gateway evaluation model");
+ expect(JSON.stringify(model)).not.toContain("account-token");
+ expect(mocks.gateway).not.toHaveBeenCalled();
+ upstream.mockResolvedValue(
+ Response.json({
+ answers: { route: { type: "choice", choice: "fast" } },
+ usage: { inputTokens: 4, outputTokens: 1 },
+ }),
+ );
+
+ const result = await model.doEvaluate({
+ state: "Alice requests a routine summary.",
+ questions: {
+ route: {
+ type: "choice",
+ instructions: "Choose a model.",
+ criteria: { fast: "Routine work", thorough: "Difficult work" },
+ },
+ },
+ });
+
+ expect(result.answers.route).toEqual({ type: "choice", choice: "fast" });
+ const [url, init] = upstream.mock.calls[0]!;
+ expect(String(url)).toBe("https://ai-gateway.vercel.sh/v4/ai/evaluation-model");
+ expect(init.headers.get("authorization")).toBe("Bearer account-token");
+ expect(init.headers.get("x-vercel-ai-gateway-team")).toBe("team_a");
+ expect(init.headers.get("ai-model-id")).toBe("typesafe-ai/jev");
+ expect(init.headers.has(DEVELOPMENT_WORKFLOW_TRANSPORT_HEADER)).toBe(false);
+ expect(init.body).not.toContain("account-token");
+});
+
it("cancels credential lookup with the model request", async () => {
const controller = new AbortController();
controller.abort(new Error("request-cancelled"));
diff --git a/packages/eve/src/internal/model-auth/transport.ts b/packages/eve/src/internal/model-auth/transport.ts
index 72c604b2a..19597a6ca 100644
--- a/packages/eve/src/internal/model-auth/transport.ts
+++ b/packages/eve/src/internal/model-auth/transport.ts
@@ -1,4 +1,8 @@
-import { createGateway, type LanguageModel } from "ai";
+import {
+ createGateway,
+ type Experimental_EvaluationModel as EvaluationModel,
+ type LanguageModel,
+} from "ai";
import { isEveDevEnvironment } from "#internal/application/dev-environment.js";
import { resolveModelApiKey } from "./api-key.js";
import {
@@ -25,8 +29,7 @@ export function createDirectModelFetch(provider: "openai" | "anthropic"): typeof
};
}
-/** Resolve the connection for every request, including models constructed before /login. */
-export function localGatewayModel(id: string): LanguageModel | undefined {
+function createLocalGateway(): ReturnType<typeof createGateway> | undefined {
if (!isEveDevEnvironment()) return undefined;
const selected = process.env[MODEL_CONNECTION_ENV];
if (
@@ -62,5 +65,17 @@ export function localGatewayModel(id: string): LanguageModel | undefined {
authenticate(await resolve(credential.token, init?.signal));
return fetch(url, { ...init, headers });
},
- })(id);
+ });
+}
+
+/** Resolve the connection for every request, including models constructed before /login. */
+export function localGatewayModel(id: string): LanguageModel | undefined {
+ return createLocalGateway()?.languageModel(id);
+}
+
+/** Resolve an evaluation model through the same local Gateway connection used by language models. */
+export function localGatewayEvaluationModel(
+ id: string,
+): Exclude<EvaluationModel, string> | undefined {
+ return createLocalGateway()?.evaluationModel(id);
}
diff --git a/research/jev-decision-models.md b/research/jev-decision-models.md
index ec630cde6..aa6d29ed0 100644
--- a/research/jev-decision-models.md
+++ b/research/jev-decision-models.md
@@ -11,7 +11,7 @@ last_updated: "2026-09-16"
Add only `autoModel` at `eve/experimental/evaluate`. Build it on AI SDK's
`experimental_evaluate` API instead of owning a TypeSafe client, AI Gateway
transport, credential resolver, or public decision API. Default to the
-`typesafe-ai/jev-latest` model string. AI SDK model strings use its default
+`typesafe-ai/jev` model string. AI SDK model strings use its default
provider, normally Vercel AI Gateway; explicit model instances use the provider
package that created them.
@@ -121,7 +121,8 @@ evaluation model adapters that use structured language-model output. Those
adapters evaluate all questions in one prompt, do not reproduce Jev's native
independent-question execution, and omit Choice and Score distributions.
-The AI SDK documentation uses `typesafe-ai/jev-latest` as the Gateway model ID.
+The [Gateway model catalog](https://vercel.com/ai-gateway/models/jev) uses
+`typesafe-ai/jev` as the Gateway model ID.
Using it as an AI SDK string keeps Gateway API keys, Vercel OIDC, request headers,
endpoint selection, and future protocol changes inside AI SDK. A direct TypeSafe
setup installs `@ai-sdk/typesafe-ai` and supplies its model instance. eve does not