🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(evals): make model preflight schema explicit (#18772)
Tobias Wochinger4718f90ed513
diff --git a/packages/shared/src/server/llm/testModelCall.test.ts b/packages/shared/src/server/llm/testModelCall.test.tsindex 21d574c460..39d4807fdb 100644--- a/packages/shared/src/server/llm/testModelCall.test.ts+++ b/packages/shared/src/server/llm/testModelCall.test.ts@@ -1,6 +1,7 @@import { createOpenAI } from "@ai-sdk/openai";import { MockLanguageModelV4 } from "ai/test";import { describe, expect, it, vi } from "vitest";+import { z } from "zod";import { encrypt } from "../../encryption";import { testModelCall } from "./testModelCall";@@ -9,6 +10,23 @@ import { LLMAdapter } from "./types";vi.mock("@ai-sdk/openai", () => ({ createOpenAI: vi.fn() }));describe("testModelCall", () => {+ const apiKey = {+ id: "api-key-id",+ projectId: "project-id",+ createdAt: new Date(),+ updatedAt: new Date(),+ adapter: LLMAdapter.OpenAI,+ provider: "openai",+ displaySecretKey: "sk-...test",+ secretKey: encrypt("sk-test"),+ extraHeaders: null,+ extraHeaderKeys: [],+ baseURL: null,+ customModels: [],+ withDefaultModels: true,+ config: null,+ };+it("rejects when the model produces no structured output", async () => {vi.mocked(createOpenAI).mockReturnValue({chat: () =>@@ -35,23 +53,58 @@ describe("testModelCall", () => {testModelCall({provider: "openai",model: "reasoning-model",- apiKey: {- id: "api-key-id",- projectId: "project-id",- createdAt: new Date(),- updatedAt: new Date(),- adapter: LLMAdapter.OpenAI,- provider: "openai",- displaySecretKey: "sk-...test",- secretKey: encrypt("sk-test"),- extraHeaders: null,- extraHeaderKeys: [],- baseURL: null,- customModels: [],- withDefaultModels: true,- config: null,- },+ apiKey,}),).rejects.toMatchObject({ name: "AI_NoOutputGeneratedError" });});++ it("shows the output schema to the model", async () => {+ const requests: Array<{ prompt: unknown }> = [];+ const model = new MockLanguageModelV4({+ doGenerate: async (options) => {+ requests.push({ prompt: options.prompt });+ return {+ content: [+ {+ type: "text" as const,+ text: '{"reasoning":"The criteria are satisfied.","score":true}',+ },+ ],+ finishReason: { unified: "stop" as const, raw: "stop" },+ usage: {+ inputTokens: {+ total: 1,+ noCache: 1,+ cacheRead: undefined,+ cacheWrite: undefined,+ },+ outputTokens: { total: 1, text: 1, reasoning: undefined },+ },+ warnings: [],+ };+ },+ });+ vi.mocked(createOpenAI).mockReturnValue({+ chat: () => model,+ responses: vi.fn(),+ } as never);++ const params = {+ provider: "openai",+ model: "reasoning-model",+ apiKey,+ structuredOutputSchema: z.object({+ reasoning: z.string().describe("Return one concise sentence."),+ score: z+ .boolean()+ .describe("Return true when the criteria are satisfied."),+ }),+ };++ await testModelCall(params);++ const prompt = JSON.stringify(requests[0]?.prompt);+ expect(prompt).toContain("Return one concise sentence.");+ expect(prompt).toContain("Return true when the criteria are satisfied.");+ });});diff --git a/packages/shared/src/server/llm/testModelCall.ts b/packages/shared/src/server/llm/testModelCall.tsindex 753dcc7be2..7885a31a4d 100644--- a/packages/shared/src/server/llm/testModelCall.ts+++ b/packages/shared/src/server/llm/testModelCall.ts@@ -35,6 +35,10 @@ export const testModelCall = async ({score: z.string(),reasoning: z.string(),});+ const promptSchema =+ schema instanceof Object && "~standard" in schema+ ? z.toJSONSchema(schema as ZodType)+ : schema;const result = await generateLLMText({...mapLegacyLLMCompletionParams({@@ -42,8 +46,10 @@ export const testModelCall = async ({messages: [{role: ChatMessageRole.User,- content:- "Generate one example response that conforms to the provided JSON schema. Respect the type, allowed values, and bounds of every field. Keep any reasoning brief.",+ content: `Generate one example response that conforms to this JSON schema. Respect the type, description, allowed values, and bounds of every field. Keep any reasoning brief.++JSON schema:+${JSON.stringify(promptSchema)}`,type: ChatMessageType.User,},],