Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(evals): make model preflight schema explicit (#18772)

Tobias Wochinger4718f90ed513
diff --git a/packages/shared/src/server/llm/testModelCall.test.ts b/packages/shared/src/server/llm/testModelCall.test.ts
index 21d574c460..39d4807fdb 100644
--- a/packages/shared/src/server/llm/testModelCall.test.ts
+++ b/packages/shared/src/server/llm/testModelCall.test.ts
@@ -1,6 +1,7 @@
import { createOpenAI } from "@ai-sdk/openai";
import { MockLanguageModelV4 } from "ai/test";
import { describe, expect, it, vi } from "vitest";
+import { z } from "zod";
import { encrypt } from "../../encryption";
import { testModelCall } from "./testModelCall";
@@ -9,6 +10,23 @@ import { LLMAdapter } from "./types";
vi.mock("@ai-sdk/openai", () => ({ createOpenAI: vi.fn() }));
describe("testModelCall", () => {
+ const apiKey = {
+ id: "api-key-id",
+ projectId: "project-id",
+ createdAt: new Date(),
+ updatedAt: new Date(),
+ adapter: LLMAdapter.OpenAI,
+ provider: "openai",
+ displaySecretKey: "sk-...test",
+ secretKey: encrypt("sk-test"),
+ extraHeaders: null,
+ extraHeaderKeys: [],
+ baseURL: null,
+ customModels: [],
+ withDefaultModels: true,
+ config: null,
+ };
+
it("rejects when the model produces no structured output", async () => {
vi.mocked(createOpenAI).mockReturnValue({
chat: () =>
@@ -35,23 +53,58 @@ describe("testModelCall", () => {
testModelCall({
provider: "openai",
model: "reasoning-model",
- apiKey: {
- id: "api-key-id",
- projectId: "project-id",
- createdAt: new Date(),
- updatedAt: new Date(),
- adapter: LLMAdapter.OpenAI,
- provider: "openai",
- displaySecretKey: "sk-...test",
- secretKey: encrypt("sk-test"),
- extraHeaders: null,
- extraHeaderKeys: [],
- baseURL: null,
- customModels: [],
- withDefaultModels: true,
- config: null,
- },
+ apiKey,
}),
).rejects.toMatchObject({ name: "AI_NoOutputGeneratedError" });
});
+
+ it("shows the output schema to the model", async () => {
+ const requests: Array<{ prompt: unknown }> = [];
+ const model = new MockLanguageModelV4({
+ doGenerate: async (options) => {
+ requests.push({ prompt: options.prompt });
+ return {
+ content: [
+ {
+ type: "text" as const,
+ text: '{"reasoning":"The criteria are satisfied.","score":true}',
+ },
+ ],
+ finishReason: { unified: "stop" as const, raw: "stop" },
+ usage: {
+ inputTokens: {
+ total: 1,
+ noCache: 1,
+ cacheRead: undefined,
+ cacheWrite: undefined,
+ },
+ outputTokens: { total: 1, text: 1, reasoning: undefined },
+ },
+ warnings: [],
+ };
+ },
+ });
+ vi.mocked(createOpenAI).mockReturnValue({
+ chat: () => model,
+ responses: vi.fn(),
+ } as never);
+
+ const params = {
+ provider: "openai",
+ model: "reasoning-model",
+ apiKey,
+ structuredOutputSchema: z.object({
+ reasoning: z.string().describe("Return one concise sentence."),
+ score: z
+ .boolean()
+ .describe("Return true when the criteria are satisfied."),
+ }),
+ };
+
+ await testModelCall(params);
+
+ const prompt = JSON.stringify(requests[0]?.prompt);
+ expect(prompt).toContain("Return one concise sentence.");
+ expect(prompt).toContain("Return true when the criteria are satisfied.");
+ });
});
diff --git a/packages/shared/src/server/llm/testModelCall.ts b/packages/shared/src/server/llm/testModelCall.ts
index 753dcc7be2..7885a31a4d 100644
--- a/packages/shared/src/server/llm/testModelCall.ts
+++ b/packages/shared/src/server/llm/testModelCall.ts
@@ -35,6 +35,10 @@ export const testModelCall = async ({
score: z.string(),
reasoning: z.string(),
});
+ const promptSchema =
+ schema instanceof Object && "~standard" in schema
+ ? z.toJSONSchema(schema as ZodType)
+ : schema;
const result = await generateLLMText({
...mapLegacyLLMCompletionParams({
@@ -42,8 +46,10 @@ export const testModelCall = async ({
messages: [
{
role: ChatMessageRole.User,
- content:
- "Generate one example response that conforms to the provided JSON schema. Respect the type, allowed values, and bounds of every field. Keep any reasoning brief.",
+ content: `Generate one example response that conforms to this JSON schema. Respect the type, description, allowed values, and bounds of every field. Keep any reasoning brief.
+
+JSON schema:
+${JSON.stringify(promptSchema)}`,
type: ChatMessageType.User,
},
],