🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(evals): avoid reasoning collisions in structured output (#16843)
Tobias Wochinger162887c97e66
diff --git a/worker/src/__tests__/evalService.test.ts b/worker/src/__tests__/evalService.test.tsindex 3236c2c814..143101dafc 100644--- a/worker/src/__tests__/evalService.test.ts+++ b/worker/src/__tests__/evalService.test.ts@@ -3450,7 +3450,9 @@ Respond with JSON: {"score": <number>, "reasoning": "<explanation>"}`;params: any,) => {capturedTraceSinkParams = params.trace;- return { output: { score: 0.8, reasoning: "Good response" } };+ return {+ output: { score: 0.8, scoreExplanation: "Good response" },+ };}) as any);await evaluate({@@ -3488,7 +3490,7 @@ Respond with JSON: {"score": <number>, "reasoning": "<explanation>"}`;vi.mocked(generateLLMText).mockResolvedValueOnce({output: {score: 0,- reasoning: "The response is safe and not toxic.",+ scoreExplanation: "The response is safe and not toxic.",},} as any);diff --git a/worker/src/__tests__/network.ts b/worker/src/__tests__/network.tsindex 27a330a04e..c15242b585 100644--- a/worker/src/__tests__/network.ts+++ b/worker/src/__tests__/network.ts@@ -17,7 +17,7 @@ const DEFAULT_RESPONSE = {// extraction tool-call shape.content: JSON.stringify({score: 0,- reasoning:+ scoreExplanation:"The provided text is a harmless play on words that poses no risk of harm or offense. It is a lighthearted joke that uses wordplay to create humor without targeting or derogating any group of people.",}),},diff --git a/worker/src/features/evaluation/__tests__/evalExecutionDeps.test.ts b/worker/src/features/evaluation/__tests__/evalExecutionDeps.test.tsindex 21042a59d8..85639d3aa5 100644--- a/worker/src/features/evaluation/__tests__/evalExecutionDeps.test.ts+++ b/worker/src/features/evaluation/__tests__/evalExecutionDeps.test.ts@@ -3,16 +3,19 @@ import { z } from "zod";import { createProductionEvalExecutionDeps } from "../evalExecutionDeps";import { EXPORT_VOLUME_METRIC } from "../../../services/exportVolumeMetric";-const { mockGenerateLLMText, mockRecordIncrement } = vi.hoisted(() => ({- mockGenerateLLMText: vi.fn(),- mockRecordIncrement: vi.fn(),-}));+const { mockCreateLLMOutput, mockGenerateLLMText, mockRecordIncrement } =+ vi.hoisted(() => ({+ mockCreateLLMOutput: vi.fn(),+ mockGenerateLLMText: vi.fn(),+ mockRecordIncrement: vi.fn(),+ }));vi.mock("@langfuse/shared/src/server", async (importOriginal) => {const original =await importOriginal<typeof import("@langfuse/shared/src/server")>();return {...original,+ createLLMOutput: mockCreateLLMOutput,generateLLMText: mockGenerateLLMText,recordIncrement: mockRecordIncrement,};@@ -32,6 +35,7 @@ vi.mock("../../../env", async (importOriginal) => {describe("createProductionEvalExecutionDeps", () => {beforeEach(() => {vi.clearAllMocks();+ mockCreateLLMOutput.mockImplementation((schema) => ({ schema }));mockGenerateLLMText.mockResolvedValue({ output: { completion: "ok" } });});@@ -56,7 +60,10 @@ describe("createProductionEvalExecutionDeps", () => {adapter: "openai" as any,modelParams: {},},- structuredOutputSchema: {} as any,+ structuredOutputSchema: z.object({+ reasoning: z.string(),+ score: z.number(),+ }),traceSinkParams: {targetProjectId: "project-123",traceId: "trace-123",@@ -81,6 +88,63 @@ describe("createProductionEvalExecutionDeps", () => {);});+ it("renames reasoning only for the model call and maps it back", async () => {+ mockGenerateLLMText.mockResolvedValue({+ output: { score: 0.8, scoreExplanation: "Good response" },+ });+ const deps = createProductionEvalExecutionDeps();+ const structuredOutputSchema = z.object({+ reasoning: z.string().describe("why this score was given"),+ score: z.number().describe("score between 0 and 1"),+ });++ const result = await deps.callLLM({+ messages: [+ {+ role: "user",+ type: "user",+ content: "Judge this answer",+ },+ ],+ modelConfig: {+ provider: "openai",+ model: "gpt-4.1",+ apiKey: { adapter: "openai", secretKey: "secret" },+ adapter: "openai" as any,+ modelParams: {},+ },+ structuredOutputSchema,+ traceSinkParams: {+ targetProjectId: "project-123",+ traceId: "trace-123",+ traceName: "Judge trace",+ environment: "langfuse-llm-as-a-judge",+ metadata: {},+ },+ });++ const modelOutput = mockGenerateLLMText.mock.calls[0][0].output as {+ schema: z.ZodType;+ };+ expect(z.toJSONSchema(modelOutput.schema)).toMatchObject({+ properties: {+ scoreExplanation: { type: "string" },+ score: { type: "number" },+ },+ required: ["scoreExplanation", "score"],+ });+ expect(z.toJSONSchema(modelOutput.schema).properties).not.toHaveProperty(+ "reasoning",+ );+ expect(+ structuredOutputSchema.safeParse({+ score: 0.8,+ reasoning: "Good response",+ }).success,+ ).toBe(true);+ expect(result).toEqual({ score: 0.8, reasoning: "Good response" });+ });+it("records llmaj export volume using the schema's JSON Schema form", async () => {const deps = createProductionEvalExecutionDeps();@@ -112,10 +176,14 @@ describe("createProductionEvalExecutionDeps", () => {},});+ const modelFacingSchema = z.object({+ scoreExplanation: structuredOutputSchema.shape.reasoning,+ score: structuredOutputSchema.shape.score,+ });const expectedBytes =Buffer.byteLength(JSON.stringify(messages), "utf8") +Buffer.byteLength(- JSON.stringify(z.toJSONSchema(structuredOutputSchema)),+ JSON.stringify(z.toJSONSchema(modelFacingSchema)),"utf8",);@@ -127,7 +195,7 @@ describe("createProductionEvalExecutionDeps", () => {);// Not the Zod _def form.const zodDefBytes = Buffer.byteLength(- JSON.stringify(structuredOutputSchema),+ JSON.stringify(modelFacingSchema),"utf8",);expect(expectedBytes).not.toBe(diff --git a/worker/src/features/evaluation/evalExecutionDeps.ts b/worker/src/features/evaluation/evalExecutionDeps.tsindex 7f751bb2e9..75be937d83 100644--- a/worker/src/features/evaluation/evalExecutionDeps.ts+++ b/worker/src/features/evaluation/evalExecutionDeps.ts@@ -19,7 +19,10 @@ import { getEvalS3StorageClient } from "./s3StorageClient";import { createInternalEventsWriter } from "../internal-tracing/createInternalEventsWriter";import { recordExportVolume } from "../../services/exportVolumeMetric";-type StructuredOutputSchema = z.ZodType;+type StructuredOutputSchema = z.ZodObject<{+ reasoning: z.ZodString;+ score: z.ZodType;+}>;/*** Result of fetching model configuration.@@ -218,15 +221,20 @@ export function createProductionEvalExecutionDeps(): EvalExecutionDeps {typeof mapLegacyLLMCompletionParams>[0]["modelParams"]["adapter"];+ // Keep the evaluator contract unchanged, but avoid exposing the+ // overloaded `reasoning` field name to the model.+ const modelFacingStructuredOutputSchema = z.object({+ scoreExplanation: params.structuredOutputSchema.shape.reasoning,+ score: params.structuredOutputSchema.shape.score,+ });+// llmaj egress: serialized request body (messages + schema), uncompressed.const bytes =Buffer.byteLength(JSON.stringify(params.messages), "utf8") +- (params.structuredOutputSchema- ? Buffer.byteLength(- serializeSchemaForEgress(params.structuredOutputSchema),- "utf8",- )- : 0);+ Buffer.byteLength(+ serializeSchemaForEgress(modelFacingStructuredOutputSchema),+ "utf8",+ );const llmParams = mapLegacyLLMCompletionParams({connection,@@ -240,7 +248,7 @@ export function createProductionEvalExecutionDeps(): EvalExecutionDeps {});const result = await generateLLMText({...llmParams,- output: createLLMOutput(params.structuredOutputSchema),+ output: createLLMOutput(modelFacingStructuredOutputSchema),maxRetries: 1,trace: {targetProjectId: params.traceSinkParams.targetProjectId,@@ -259,7 +267,10 @@ export function createProductionEvalExecutionDeps(): EvalExecutionDeps {projectId: params.traceSinkParams.targetProjectId,});- return result.output;+ return {+ score: result.output.score,+ reasoning: result.output.scoreExplanation,+ };},fetchModelConfig: async ({ projectId, provider, model, modelParams }) => {