Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(evals): avoid reasoning collisions in structured output (#16843)

Tobias Wochinger162887c97e66
diff --git a/worker/src/__tests__/evalService.test.ts b/worker/src/__tests__/evalService.test.ts
index 3236c2c814..143101dafc 100644
--- a/worker/src/__tests__/evalService.test.ts
+++ b/worker/src/__tests__/evalService.test.ts
@@ -3450,7 +3450,9 @@ Respond with JSON: {"score": <number>, "reasoning": "<explanation>"}`;
params: any,
) => {
capturedTraceSinkParams = params.trace;
- return { output: { score: 0.8, reasoning: "Good response" } };
+ return {
+ output: { score: 0.8, scoreExplanation: "Good response" },
+ };
}) as any);
await evaluate({
@@ -3488,7 +3490,7 @@ Respond with JSON: {"score": <number>, "reasoning": "<explanation>"}`;
vi.mocked(generateLLMText).mockResolvedValueOnce({
output: {
score: 0,
- reasoning: "The response is safe and not toxic.",
+ scoreExplanation: "The response is safe and not toxic.",
},
} as any);
diff --git a/worker/src/__tests__/network.ts b/worker/src/__tests__/network.ts
index 27a330a04e..c15242b585 100644
--- a/worker/src/__tests__/network.ts
+++ b/worker/src/__tests__/network.ts
@@ -17,7 +17,7 @@ const DEFAULT_RESPONSE = {
// extraction tool-call shape.
content: JSON.stringify({
score: 0,
- reasoning:
+ scoreExplanation:
"The provided text is a harmless play on words that poses no risk of harm or offense. It is a lighthearted joke that uses wordplay to create humor without targeting or derogating any group of people.",
}),
},
diff --git a/worker/src/features/evaluation/__tests__/evalExecutionDeps.test.ts b/worker/src/features/evaluation/__tests__/evalExecutionDeps.test.ts
index 21042a59d8..85639d3aa5 100644
--- a/worker/src/features/evaluation/__tests__/evalExecutionDeps.test.ts
+++ b/worker/src/features/evaluation/__tests__/evalExecutionDeps.test.ts
@@ -3,16 +3,19 @@ import { z } from "zod";
import { createProductionEvalExecutionDeps } from "../evalExecutionDeps";
import { EXPORT_VOLUME_METRIC } from "../../../services/exportVolumeMetric";
-const { mockGenerateLLMText, mockRecordIncrement } = vi.hoisted(() => ({
- mockGenerateLLMText: vi.fn(),
- mockRecordIncrement: vi.fn(),
-}));
+const { mockCreateLLMOutput, mockGenerateLLMText, mockRecordIncrement } =
+ vi.hoisted(() => ({
+ mockCreateLLMOutput: vi.fn(),
+ mockGenerateLLMText: vi.fn(),
+ mockRecordIncrement: vi.fn(),
+ }));
vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
const original =
await importOriginal<typeof import("@langfuse/shared/src/server")>();
return {
...original,
+ createLLMOutput: mockCreateLLMOutput,
generateLLMText: mockGenerateLLMText,
recordIncrement: mockRecordIncrement,
};
@@ -32,6 +35,7 @@ vi.mock("../../../env", async (importOriginal) => {
describe("createProductionEvalExecutionDeps", () => {
beforeEach(() => {
vi.clearAllMocks();
+ mockCreateLLMOutput.mockImplementation((schema) => ({ schema }));
mockGenerateLLMText.mockResolvedValue({ output: { completion: "ok" } });
});
@@ -56,7 +60,10 @@ describe("createProductionEvalExecutionDeps", () => {
adapter: "openai" as any,
modelParams: {},
},
- structuredOutputSchema: {} as any,
+ structuredOutputSchema: z.object({
+ reasoning: z.string(),
+ score: z.number(),
+ }),
traceSinkParams: {
targetProjectId: "project-123",
traceId: "trace-123",
@@ -81,6 +88,63 @@ describe("createProductionEvalExecutionDeps", () => {
);
});
+ it("renames reasoning only for the model call and maps it back", async () => {
+ mockGenerateLLMText.mockResolvedValue({
+ output: { score: 0.8, scoreExplanation: "Good response" },
+ });
+ const deps = createProductionEvalExecutionDeps();
+ const structuredOutputSchema = z.object({
+ reasoning: z.string().describe("why this score was given"),
+ score: z.number().describe("score between 0 and 1"),
+ });
+
+ const result = await deps.callLLM({
+ messages: [
+ {
+ role: "user",
+ type: "user",
+ content: "Judge this answer",
+ },
+ ],
+ modelConfig: {
+ provider: "openai",
+ model: "gpt-4.1",
+ apiKey: { adapter: "openai", secretKey: "secret" },
+ adapter: "openai" as any,
+ modelParams: {},
+ },
+ structuredOutputSchema,
+ traceSinkParams: {
+ targetProjectId: "project-123",
+ traceId: "trace-123",
+ traceName: "Judge trace",
+ environment: "langfuse-llm-as-a-judge",
+ metadata: {},
+ },
+ });
+
+ const modelOutput = mockGenerateLLMText.mock.calls[0][0].output as {
+ schema: z.ZodType;
+ };
+ expect(z.toJSONSchema(modelOutput.schema)).toMatchObject({
+ properties: {
+ scoreExplanation: { type: "string" },
+ score: { type: "number" },
+ },
+ required: ["scoreExplanation", "score"],
+ });
+ expect(z.toJSONSchema(modelOutput.schema).properties).not.toHaveProperty(
+ "reasoning",
+ );
+ expect(
+ structuredOutputSchema.safeParse({
+ score: 0.8,
+ reasoning: "Good response",
+ }).success,
+ ).toBe(true);
+ expect(result).toEqual({ score: 0.8, reasoning: "Good response" });
+ });
+
it("records llmaj export volume using the schema's JSON Schema form", async () => {
const deps = createProductionEvalExecutionDeps();
@@ -112,10 +176,14 @@ describe("createProductionEvalExecutionDeps", () => {
},
});
+ const modelFacingSchema = z.object({
+ scoreExplanation: structuredOutputSchema.shape.reasoning,
+ score: structuredOutputSchema.shape.score,
+ });
const expectedBytes =
Buffer.byteLength(JSON.stringify(messages), "utf8") +
Buffer.byteLength(
- JSON.stringify(z.toJSONSchema(structuredOutputSchema)),
+ JSON.stringify(z.toJSONSchema(modelFacingSchema)),
"utf8",
);
@@ -127,7 +195,7 @@ describe("createProductionEvalExecutionDeps", () => {
);
// Not the Zod _def form.
const zodDefBytes = Buffer.byteLength(
- JSON.stringify(structuredOutputSchema),
+ JSON.stringify(modelFacingSchema),
"utf8",
);
expect(expectedBytes).not.toBe(
diff --git a/worker/src/features/evaluation/evalExecutionDeps.ts b/worker/src/features/evaluation/evalExecutionDeps.ts
index 7f751bb2e9..75be937d83 100644
--- a/worker/src/features/evaluation/evalExecutionDeps.ts
+++ b/worker/src/features/evaluation/evalExecutionDeps.ts
@@ -19,7 +19,10 @@ import { getEvalS3StorageClient } from "./s3StorageClient";
import { createInternalEventsWriter } from "../internal-tracing/createInternalEventsWriter";
import { recordExportVolume } from "../../services/exportVolumeMetric";
-type StructuredOutputSchema = z.ZodType;
+type StructuredOutputSchema = z.ZodObject<{
+ reasoning: z.ZodString;
+ score: z.ZodType;
+}>;
/**
* Result of fetching model configuration.
@@ -218,15 +221,20 @@ export function createProductionEvalExecutionDeps(): EvalExecutionDeps {
typeof mapLegacyLLMCompletionParams
>[0]["modelParams"]["adapter"];
+ // Keep the evaluator contract unchanged, but avoid exposing the
+ // overloaded `reasoning` field name to the model.
+ const modelFacingStructuredOutputSchema = z.object({
+ scoreExplanation: params.structuredOutputSchema.shape.reasoning,
+ score: params.structuredOutputSchema.shape.score,
+ });
+
// llmaj egress: serialized request body (messages + schema), uncompressed.
const bytes =
Buffer.byteLength(JSON.stringify(params.messages), "utf8") +
- (params.structuredOutputSchema
- ? Buffer.byteLength(
- serializeSchemaForEgress(params.structuredOutputSchema),
- "utf8",
- )
- : 0);
+ Buffer.byteLength(
+ serializeSchemaForEgress(modelFacingStructuredOutputSchema),
+ "utf8",
+ );
const llmParams = mapLegacyLLMCompletionParams({
connection,
@@ -240,7 +248,7 @@ export function createProductionEvalExecutionDeps(): EvalExecutionDeps {
});
const result = await generateLLMText({
...llmParams,
- output: createLLMOutput(params.structuredOutputSchema),
+ output: createLLMOutput(modelFacingStructuredOutputSchema),
maxRetries: 1,
trace: {
targetProjectId: params.traceSinkParams.targetProjectId,
@@ -259,7 +267,10 @@ export function createProductionEvalExecutionDeps(): EvalExecutionDeps {
projectId: params.traceSinkParams.targetProjectId,
});
- return result.output;
+ return {
+ score: result.output.score,
+ reasoning: result.output.scoreExplanation,
+ };
},
fetchModelConfig: async ({ projectId, provider, model, modelParams }) => {