🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(evals): send test evaluator runs through the live judge call (#18816)
Tobias Wochinger4e5adc805901
diff --git a/packages/shared/src/server/evals/llmEvaluatorExecution.test.ts b/packages/shared/src/server/evals/llmEvaluatorExecution.test.tsindex 12d6da991a..a31506653f 100644--- a/packages/shared/src/server/evals/llmEvaluatorExecution.test.ts+++ b/packages/shared/src/server/evals/llmEvaluatorExecution.test.ts@@ -1,6 +1,100 @@-import { describe, expect, it, vi } from "vitest";-import { createNumericEvalOutputDefinition } from "../../features/evals/outputDefinition";-import { executeLlmEvaluator } from "./llmEvaluatorExecution";+import { beforeEach, describe, expect, it, vi } from "vitest";+import { z } from "zod";+import {+ compilePersistedEvalOutputDefinition,+ createNumericEvalOutputDefinition,+} from "../../features/evals/outputDefinition";+import { ChatMessageRole, ChatMessageType, LLMAdapter } from "../llm/types";+import {+ executeLlmEvaluator,+ generateLlmEvaluatorOutput,+} from "./llmEvaluatorExecution";++const { mockGenerateLLMText } = vi.hoisted(() => ({+ mockGenerateLLMText: vi.fn(),+}));++vi.mock("../llm/llmText", async (importOriginal) => ({+ ...(await importOriginal<typeof import("../llm/llmText")>()),+ createLLMOutput: (schema: z.ZodType) => ({ schema }),+ generateLLMText: mockGenerateLLMText,+}));++vi.mock("../llm/mediaMessages", () => ({+ compileLangfuseMediaMessages: async ({+ messages,+ }: {+ messages: Array<{ role: string; content: string }>;+ }) => {+ const mapped = messages.map(({ role, content }) => ({ role, content }));+ return { providerMessages: mapped, traceMessages: mapped };+ },+}));++describe("generateLlmEvaluatorOutput", () => {+ beforeEach(() => {+ mockGenerateLLMText.mockReset();+ });++ it("renames reasoning only for the model call and maps it back", async () => {+ mockGenerateLLMText.mockResolvedValue({+ output: { score: 0.8, scoreExplanation: "Good response" },+ usage: { inputTokens: 10, outputTokens: 5 },+ });+ const { outputResultSchema } = compilePersistedEvalOutputDefinition(+ createNumericEvalOutputDefinition({+ scoreDescription: "score between 0 and 1",+ reasoningDescription: "why this score was given",+ }),+ );++ const result = await generateLlmEvaluatorOutput({+ projectId: "project-123",+ connection: { secretKey: "secret" },+ modelParams: {+ provider: "openai",+ model: "gpt-4.1",+ adapter: LLMAdapter.OpenAI,+ },+ messages: [+ {+ role: ChatMessageRole.User,+ type: ChatMessageType.User,+ content: "Judge this answer",+ },+ ],+ outputResultSchema,+ trace: {+ targetProjectId: "project-123",+ traceId: "trace-123",+ traceName: "Judge trace",+ environment: "langfuse-llm-as-a-judge",+ },+ });++ const call = mockGenerateLLMText.mock.calls[0][0];+ expect(call).toMatchObject({+ maxRetries: 1,+ trace: { traceId: "trace-123" },+ });+ const modelSchema = z.toJSONSchema(call.output.schema as z.ZodType);+ expect(modelSchema).toMatchObject({+ description:+ 'Return only top-level "score" and "scoreExplanation". Put other requested fields inside "scoreExplanation".',+ properties: {+ scoreExplanation: {+ type: "string",+ description: "why this score was given",+ },+ score: { type: "number", description: "score between 0 and 1" },+ },+ required: ["scoreExplanation", "score"],+ });+ expect(modelSchema.properties).not.toHaveProperty("reasoning");+ expect(result.output).toEqual({ score: 0.8, reasoning: "Good response" });+ expect(result.usage).toEqual({ inputTokens: 10, outputTokens: 5 });+ });+});describe("executeLlmEvaluator", () => {it("interpolates a single user prompt message", async () => {diff --git a/packages/shared/src/server/evals/llmEvaluatorExecution.ts b/packages/shared/src/server/evals/llmEvaluatorExecution.tsindex baddabffdf..c388bd1f1f 100644--- a/packages/shared/src/server/evals/llmEvaluatorExecution.ts+++ b/packages/shared/src/server/evals/llmEvaluatorExecution.ts@@ -14,7 +14,20 @@ import {ChatMessageRole,ChatMessageType,type ChatMessage,+ type ModelParams,+ type TraceSinkParams,} from "../llm/types";+import {+ createLLMOutput,+ generateLLMText,+ mapLegacyLLMCompletionParams,+ type LLMConnection,+} from "../llm/llmText";+import { compileLangfuseMediaMessages } from "../llm/mediaMessages";+import { z } from "zod";++const MODEL_FACING_OUTPUT_SCHEMA_DESCRIPTION =+ 'Return only top-level "score" and "scoreExplanation". Put other requested fields inside "scoreExplanation".';const CHAT_MESSAGE_BUILDERS = {system: (content: string) => ({@@ -56,6 +69,60 @@ function buildEvalMessages(params: {};}+/**+ * The single model call behind live and test LLM-as-a-judge runs, so a test+ * run sends the provider exactly what a live run sends.+ *+ * The model sees `scoreExplanation` instead of `reasoning`: thinking-capable+ * models treat a `reasoning` field as their scratchpad and drop the score.+ * The output is mapped back to the `{ score, reasoning }` evaluator contract.+ */+export async function generateLlmEvaluatorOutput(params: {+ projectId: string;+ connection: LLMConnection;+ modelParams: ModelParams;+ messages: ChatMessage[];+ outputResultSchema: z.ZodObject<{ reasoning: z.ZodString; score: z.ZodType }>;+ trace: TraceSinkParams;+}) {+ const llmParams = mapLegacyLLMCompletionParams({+ connection: params.connection,+ messages: params.messages,+ modelParams: params.modelParams,+ });+ const { providerMessages, traceMessages } =+ await compileLangfuseMediaMessages({+ projectId: params.projectId,+ messages: params.messages,+ adapter: params.modelParams.adapter,+ });+ const modelFacingOutputSchema = z+ .object({+ scoreExplanation: params.outputResultSchema.shape.reasoning,+ score: params.outputResultSchema.shape.score,+ })+ .describe(MODEL_FACING_OUTPUT_SCHEMA_DESCRIPTION);++ const result = await generateLLMText({+ ...llmParams,+ messages: providerMessages,+ traceInput: traceMessages,+ output: createLLMOutput(modelFacingOutputSchema),+ maxRetries: 1,+ trace: params.trace,+ });++ return {+ output: {+ score: result.output.score,+ reasoning: result.output.scoreExplanation,+ },+ usage: result.usage,+ providerMessages,+ modelFacingOutputSchema,+ };+}+export async function executeLlmEvaluator(params: {promptMessages: PersistedEvaluatorPromptMessages;variables: ExtractedVariable[];diff --git a/web/src/__tests__/server/unit/evaluator-v2-test-evaluator.servertest.ts b/web/src/__tests__/server/unit/evaluator-v2-test-evaluator.servertest.tsindex b2e171da63..b5d97fd4be 100644--- a/web/src/__tests__/server/unit/evaluator-v2-test-evaluator.servertest.ts+++ b/web/src/__tests__/server/unit/evaluator-v2-test-evaluator.servertest.ts@@ -3,10 +3,9 @@ import {LLMAdapter,} from "@langfuse/shared";import {- compileLangfuseMediaMessages,DefaultEvalModelService,findModel,- generateLLMText,+ generateLlmEvaluatorOutput,resolveConfiguredCodeEvalDispatcher,runCodeBasedEvaluationDispatch,} from "@langfuse/shared/src/server";@@ -23,8 +22,7 @@ vi.mock("@langfuse/shared/src/server", async () => ({DefaultEvalModelService: {fetchValidModelConfig: vi.fn(),},- compileLangfuseMediaMessages: vi.fn(),- generateLLMText: vi.fn(),+ generateLlmEvaluatorOutput: vi.fn(),findModel: vi.fn(),resolveConfiguredCodeEvalDispatcher: vi.fn(),runCodeBasedEvaluationDispatch: vi.fn(),@@ -56,13 +54,7 @@ describe("testEvaluator", () => {},} as unknown as ValidModelConfig,});- vi.mocked(compileLangfuseMediaMessages).mockImplementation(- async ({ messages }) => {- const mapped = messages.map(({ role, content }) => ({ role, content }));- return { providerMessages: mapped, traceMessages: mapped } as never;- },- );- vi.mocked(generateLLMText).mockResolvedValue({+ vi.mocked(generateLlmEvaluatorOutput).mockResolvedValue({output: { score: 1, reasoning: "Correct" },usage: { inputTokens: 100, outputTokens: 50, totalTokens: 150 },} as never);@@ -139,12 +131,10 @@ describe("testEvaluator", () => {startTime,shouldReadFromObservationsTable: true,});- expect(compileLangfuseMediaMessages).toHaveBeenCalledWith(- expect.objectContaining({ projectId: "project-1" }),- );- expect(generateLLMText).toHaveBeenCalledWith(+ // Live runs send the model this same call, so Test reproduces them.+ expect(generateLlmEvaluatorOutput).toHaveBeenCalledWith(expect.objectContaining({- maxRetries: 1,+ projectId: "project-1",messages: [expect.objectContaining({role: "system",@@ -155,16 +145,12 @@ describe("testEvaluator", () => {content: "Judge It arrives tomorrow.",}),],- traceInput: [- expect.objectContaining({- role: "system",- content: "Judge consistently",+ outputResultSchema: expect.objectContaining({+ shape: expect.objectContaining({+ reasoning: expect.anything(),+ score: expect.anything(),}),- expect.objectContaining({- role: "user",- content: "Judge It arrives tomorrow.",- }),- ],+ }),trace: expect.objectContaining({evaluationContext: {evaluatorId: "evaluator-1",diff --git a/web/src/features/evals/v2/server/evaluators/testEvaluator.ts b/web/src/features/evals/v2/server/evaluators/testEvaluator.tsindex 3f8ac76f10..7a8f87cd96 100644--- a/web/src/features/evals/v2/server/evaluators/testEvaluator.ts+++ b/web/src/features/evals/v2/server/evaluators/testEvaluator.ts@@ -5,22 +5,19 @@ import {import { decrypt } from "@langfuse/shared/encryption";import {buildEvalExecutionData,- compileLangfuseMediaMessages,createDecisionModelClient,createW3CTraceId,decryptAndParseExtraHeaders,DefaultEvalModelService,- createLLMOutput,executeDecisionModelEvaluator,executeLlmEvaluator,extractObservationVariables,findModel,- generateLLMText,+ generateLlmEvaluatorOutput,isAllowedDecisionModel,LLMAdapter,OPENAI_DECISION_MODEL_IDS,LangfuseInternalTraceEnvironment,- mapLegacyLLMCompletionParams,matchPricingTier,resolveConfiguredCodeEvalDispatcher,runCodeBasedEvaluationDispatch,@@ -236,29 +233,17 @@ async function testLlmEvaluator(params: {interpolatedPrompt: prompt,}) => {interpolatedPrompt = prompt;- const modelParams = {- provider: modelConfig.config.provider,- model: modelConfig.config.model,- adapter: modelConfig.config.apiKey.adapter,- ...modelConfig.config.modelParams,- };- const llmParams = mapLegacyLLMCompletionParams({+ const result = await generateLlmEvaluatorOutput({+ projectId: params.projectId,connection: modelConfig.config.apiKey,- messages,- modelParams,- });- const { providerMessages, traceMessages } =- await compileLangfuseMediaMessages({- projectId: params.projectId,- messages,+ modelParams: {+ provider: modelConfig.config.provider,+ model: modelConfig.config.model,adapter: modelConfig.config.apiKey.adapter,- });- const result = await generateLLMText({- ...llmParams,- messages: providerMessages,- traceInput: traceMessages,- output: createLLMOutput(compiledOutputDefinition.outputResultSchema),- maxRetries: 1,+ ...modelConfig.config.modelParams,+ },+ messages,+ outputResultSchema: compiledOutputDefinition.outputResultSchema,trace: {targetProjectId: params.projectId,traceId: executionTraceId,diff --git a/worker/src/__tests__/evalService.test.ts b/worker/src/__tests__/evalService.test.tsindex 143101dafc..67a52e62fb 100644--- a/worker/src/__tests__/evalService.test.ts+++ b/worker/src/__tests__/evalService.test.ts@@ -33,19 +33,21 @@ import {} from "../features/evaluation/evalService";import { requiresDatabaseLookup } from "../features/evaluation/traceFilterUtils";-// Mock the shared LLM runtime with default passthrough behavior.+// Mock the shared evaluator model call with default passthrough behavior.vi.mock("@langfuse/shared/src/server", async () => {- const actual = await vi.importActual("@langfuse/shared/src/server");+ const actual = await vi.importActual<+ typeof import("@langfuse/shared/src/server")+ >("@langfuse/shared/src/server");return {...actual,- generateLLMText: vi.fn().mockImplementation(actual.generateLLMText as any),+ generateLlmEvaluatorOutput: vi.fn(actual.generateLlmEvaluatorOutput),};});// Import the mocked functionimport {EvalExecutionQueue,- generateLLMText,+ generateLlmEvaluatorOutput,} from "@langfuse/shared/src/server";import { UnrecoverableError } from "../errors/UnrecoverableError";@@ -2539,8 +2541,10 @@ Respond with JSON: {"score": <number>, "reasoning": "<explanation>"}`;test("handles LLM timeout gracefully", async () => {const { projectId } = await createOrgProjectAndApiKey();// Set up the mock to simulate timeout for this test only- const mockGenerateLLMText = vi.mocked(generateLLMText);- mockGenerateLLMText.mockRejectedValueOnce(+ const mockGenerateLlmEvaluatorOutput = vi.mocked(+ generateLlmEvaluatorOutput,+ );+ mockGenerateLlmEvaluatorOutput.mockRejectedValueOnce(new ApiError("Request timeout after 120000ms", 500),);@@ -2644,7 +2648,7 @@ Respond with JSON: {"score": <number>, "reasoning": "<explanation>"}`;expect(jobs[0].status.toString()).toBe("PENDING");// Clean up the mock after this test- mockGenerateLLMText.mockReset();+ mockGenerateLlmEvaluatorOutput.mockReset();}, 15_000);});@@ -3446,12 +3450,15 @@ Respond with JSON: {"score": <number>, "reasoning": "<explanation>"}`;// Mock the LLM runtime to capture the trace parameters.let capturedTraceSinkParams: any = null;- vi.mocked(generateLLMText).mockImplementationOnce((async (+ vi.mocked(generateLlmEvaluatorOutput).mockImplementationOnce((async (params: any,) => {capturedTraceSinkParams = params.trace;return {- output: { score: 0.8, scoreExplanation: "Good response" },+ output: { score: 0.8, reasoning: "Good response" },+ usage: {},+ providerMessages: [],+ modelFacingOutputSchema: params.outputResultSchema,};}) as any);@@ -3487,11 +3494,14 @@ Respond with JSON: {"score": <number>, "reasoning": "<explanation>"}`;const jobExecutionId = randomUUID();const templateId = randomUUID();- vi.mocked(generateLLMText).mockResolvedValueOnce({+ vi.mocked(generateLlmEvaluatorOutput).mockResolvedValueOnce({output: {score: 0,- scoreExplanation: "The response is safe and not toxic.",+ reasoning: "The response is safe and not toxic.",},+ usage: {},+ providerMessages: [],+ modelFacingOutputSchema: {},} as any);await upsertTrace({diff --git a/worker/src/features/evaluation/__tests__/evalExecutionDeps.test.ts b/worker/src/features/evaluation/__tests__/evalExecutionDeps.test.tsindex 4f0c05e759..3c23f492a9 100644--- a/worker/src/features/evaluation/__tests__/evalExecutionDeps.test.ts+++ b/worker/src/features/evaluation/__tests__/evalExecutionDeps.test.ts@@ -3,26 +3,19 @@ import { z } from "zod";import { createProductionEvalExecutionDeps } from "../evalExecutionDeps";import { EXPORT_VOLUME_METRIC } from "../../../services/exportVolumeMetric";-const {- mockCompileLangfuseMediaMessages,- mockCreateLLMOutput,- mockGenerateLLMText,- mockRecordIncrement,-} = vi.hoisted(() => ({- mockCompileLangfuseMediaMessages: vi.fn(),- mockCreateLLMOutput: vi.fn(),- mockGenerateLLMText: vi.fn(),- mockRecordIncrement: vi.fn(),-}));+const { mockGenerateLlmEvaluatorOutput, mockRecordIncrement } = vi.hoisted(+ () => ({+ mockGenerateLlmEvaluatorOutput: vi.fn(),+ mockRecordIncrement: vi.fn(),+ }),+);vi.mock("@langfuse/shared/src/server", async (importOriginal) => {const original =await importOriginal<typeof import("@langfuse/shared/src/server")>();return {...original,- compileLangfuseMediaMessages: mockCompileLangfuseMediaMessages,- createLLMOutput: mockCreateLLMOutput,- generateLLMText: mockGenerateLLMText,+ generateLlmEvaluatorOutput: mockGenerateLlmEvaluatorOutput,recordIncrement: mockRecordIncrement,};});@@ -38,64 +31,52 @@ vi.mock("../../../env", async (importOriginal) => {};});+const modelConfig = {+ provider: "openai",+ model: "gpt-4.1",+ apiKey: { adapter: "openai", secretKey: "secret" },+ adapter: "openai" as any,+ modelParams: {},+};++const traceSinkParams = {+ targetProjectId: "project-123",+ traceId: "trace-123",+ traceName: "Judge trace",+ environment: "langfuse-llm-as-a-judge",+ metadata: { score_id: "score-123" },+};++const structuredOutputSchema = z.object({+ reasoning: z.string().describe("why this score was given"),+ score: z.number().describe("score between 0 and 1"),+});+describe("createProductionEvalExecutionDeps", () => {beforeEach(() => {vi.clearAllMocks();- mockCompileLangfuseMediaMessages.mockImplementation(- async ({ messages }) => {- const mapped = messages.map(({ role, content }: any) => ({- role,- content,- }));- return { providerMessages: mapped, traceMessages: mapped };- },- );- mockCreateLLMOutput.mockImplementation((schema) => ({ schema }));- mockGenerateLLMText.mockResolvedValue({ output: { completion: "ok" } });+ mockGenerateLlmEvaluatorOutput.mockResolvedValue({+ output: { score: 0.8, reasoning: "Good response" },+ usage: {},+ providerMessages: [],+ modelFacingOutputSchema: structuredOutputSchema,+ });});- it("enables internal direct event write for llm-as-a-judge traces", async () => {+ it("calls the shared evaluator model call with direct event writes", async () => {const deps = createProductionEvalExecutionDeps();- await deps.callLLM({- messages: [- {- role: "user",- type: "user",- content: "Judge this answer",- },- ],- modelConfig: {- provider: "openai",- model: "gpt-4.1",- apiKey: {- adapter: "openai",- secretKey: "secret",- },- adapter: "openai" as any,- modelParams: {},- },- structuredOutputSchema: z.object({- reasoning: z.string(),- score: z.number(),- }),- traceSinkParams: {- targetProjectId: "project-123",- traceId: "trace-123",- traceName: "Judge trace",- environment: "langfuse-llm-as-a-judge",- metadata: {- score_id: "score-123",- },- },- });+ const result = await deps.callLLM({+ messages: [{ role: "user", type: "user", content: "Judge this answer" }],+ modelConfig,+ structuredOutputSchema,+ traceSinkParams,+ } as any);- expect(mockCompileLangfuseMediaMessages).toHaveBeenCalledWith(- expect.objectContaining({ projectId: "project-123" }),- );- expect(mockGenerateLLMText).toHaveBeenCalledWith(+ expect(mockGenerateLlmEvaluatorOutput).toHaveBeenCalledWith(expect.objectContaining({- maxRetries: 1,+ projectId: "project-123",+ outputResultSchema: structuredOutputSchema,trace: expect.objectContaining({traceId: "trace-123",environment: "langfuse-llm-as-a-judge",@@ -105,73 +86,10 @@ describe("createProductionEvalExecutionDeps", () => {}),}),);- });-- it("renames reasoning only for the model call and maps it back", async () => {- mockGenerateLLMText.mockResolvedValue({- output: { score: 0.8, scoreExplanation: "Good response" },- });- const deps = createProductionEvalExecutionDeps();- const structuredOutputSchema = z.object({- reasoning: z.string().describe("why this score was given"),- score: z.number().describe("score between 0 and 1"),- });-- const result = await deps.callLLM({- messages: [- {- role: "user",- type: "user",- content: "Judge this answer",- },- ],- modelConfig: {- provider: "openai",- model: "gpt-4.1",- apiKey: { adapter: "openai", secretKey: "secret" },- adapter: "openai" as any,- modelParams: {},- },- structuredOutputSchema,- traceSinkParams: {- targetProjectId: "project-123",- traceId: "trace-123",- traceName: "Judge trace",- environment: "langfuse-llm-as-a-judge",- metadata: {},- },- });-- const modelOutput = mockGenerateLLMText.mock.calls[0][0].output as {- schema: z.ZodType;- };- expect(z.toJSONSchema(modelOutput.schema)).toMatchObject({- description:- 'Return only top-level "score" and "scoreExplanation". Put other requested fields inside "scoreExplanation".',- properties: {- scoreExplanation: { type: "string" },- score: { type: "number" },- },- required: ["scoreExplanation", "score"],- });- expect(z.toJSONSchema(modelOutput.schema).properties).not.toHaveProperty(- "reasoning",- );- expect(- structuredOutputSchema.safeParse({- score: 0.8,- reasoning: "Good response",- }).success,- ).toBe(true);expect(result).toEqual({ score: 0.8, reasoning: "Good response" });});it("records llmaj export volume using the schema's JSON Schema form", async () => {- const deps = createProductionEvalExecutionDeps();-- const messages = [- { role: "user", type: "user", content: "Judge this answer" },- ];const providerMessages = [{role: "user",@@ -185,34 +103,27 @@ describe("createProductionEvalExecutionDeps", () => {],},];- mockCompileLangfuseMediaMessages.mockResolvedValueOnce({- providerMessages,- traceMessages: messages,- });// Production passes a Zod schema, not a plain object.- const structuredOutputSchema = z.object({- reasoning: z.string().describe("why this score was given"),- score: z.number().describe("score between 0 and 1"),+ const modelFacingSchema = z+ .object({+ scoreExplanation: structuredOutputSchema.shape.reasoning,+ score: structuredOutputSchema.shape.score,+ })+ .describe("model-facing schema");+ mockGenerateLlmEvaluatorOutput.mockResolvedValueOnce({+ output: { score: 0.8, reasoning: "Good response" },+ usage: {},+ providerMessages,+ modelFacingOutputSchema: modelFacingSchema,});+ const deps = createProductionEvalExecutionDeps();await deps.callLLM({- messages: messages as any,- modelConfig: {- provider: "openai",- model: "gpt-4.1",- apiKey: { adapter: "openai", secretKey: "secret" },- adapter: "openai" as any,- modelParams: {},- },- structuredOutputSchema: structuredOutputSchema as any,- traceSinkParams: {- targetProjectId: "project-123",- traceId: "trace-123",- traceName: "Judge trace",- environment: "langfuse-llm-as-a-judge",- metadata: {},- },- });+ messages: [{ role: "user", type: "user", content: "Judge this answer" }],+ modelConfig,+ structuredOutputSchema,+ traceSinkParams,+ } as any);const serializedProviderMessages = JSON.stringify(providerMessages,@@ -221,14 +132,6 @@ describe("createProductionEvalExecutionDeps", () => {? Buffer.from(value).toString("base64"): value,);- const modelFacingSchema = z- .object({- scoreExplanation: structuredOutputSchema.shape.reasoning,- score: structuredOutputSchema.shape.score,- })- .describe(- 'Return only top-level "score" and "scoreExplanation". Put other requested fields inside "scoreExplanation".',- );const expectedBytes =Buffer.byteLength(serializedProviderMessages, "utf8") +Buffer.byteLength(@@ -236,19 +139,32 @@ describe("createProductionEvalExecutionDeps", () => {"utf8",);- expect(expectedBytes).toBeGreaterThan(0);expect(mockRecordIncrement).toHaveBeenCalledWith(EXPORT_VOLUME_METRIC,expectedBytes,{ integration: "llmaj" },);// Not the Zod _def form.- const zodDefBytes = Buffer.byteLength(- JSON.stringify(modelFacingSchema),- "utf8",- );expect(expectedBytes).not.toBe(- Buffer.byteLength(JSON.stringify(messages), "utf8") + zodDefBytes,+ Buffer.byteLength(serializedProviderMessages, "utf8") ++ Buffer.byteLength(JSON.stringify(modelFacingSchema), "utf8"),);});++ it("does not record export volume when the model call fails", async () => {+ mockGenerateLlmEvaluatorOutput.mockRejectedValueOnce(new Error("boom"));+ const deps = createProductionEvalExecutionDeps();++ await expect(+ deps.callLLM({+ messages: [+ { role: "user", type: "user", content: "Judge this answer" },+ ],+ modelConfig,+ structuredOutputSchema,+ traceSinkParams,+ } as any),+ ).rejects.toThrow("boom");+ expect(mockRecordIncrement).not.toHaveBeenCalled();+ });});diff --git a/worker/src/features/evaluation/evalExecutionDeps.ts b/worker/src/features/evaluation/evalExecutionDeps.tsindex 6c20c134f6..4ef05e6653 100644--- a/worker/src/features/evaluation/evalExecutionDeps.ts+++ b/worker/src/features/evaluation/evalExecutionDeps.ts@@ -6,12 +6,10 @@ import { prisma } from "@langfuse/shared/src/db";import { decrypt } from "@langfuse/shared/encryption";import {buildEventBucketPrefix,- compileLangfuseMediaMessages,- createLLMOutput,createDecisionModelClient,decryptAndParseExtraHeaders,DefaultEvalModelService,- generateLLMText,+ generateLlmEvaluatorOutput,IngestionQueue,LLMAdapter,mapLegacyLLMCompletionParams,@@ -34,9 +32,6 @@ type StructuredOutputSchema = z.ZodObject<{score: z.ZodType;}>;-const MODEL_FACING_OUTPUT_SCHEMA_DESCRIPTION =- 'Return only top-level "score" and "scoreExplanation". Put other requested fields inside "scoreExplanation".';-/*** Result of fetching model configuration.*/@@ -255,51 +250,17 @@ export function createProductionEvalExecutionDeps(): EvalExecutionDeps {typeof mapLegacyLLMCompletionParams>[0]["modelParams"]["adapter"];- const modelParams = {- provider: params.modelConfig.provider,- model: params.modelConfig.model,- adapter,- ...params.modelConfig.modelParams,- };- const llmParams = mapLegacyLLMCompletionParams({+ const result = await generateLlmEvaluatorOutput({+ projectId: params.traceSinkParams.targetProjectId,connection,- messages: params.messages,- modelParams,- });- const { providerMessages, traceMessages } =- await compileLangfuseMediaMessages({- projectId: params.traceSinkParams.targetProjectId,- messages: params.messages,+ modelParams: {+ provider: params.modelConfig.provider,+ model: params.modelConfig.model,adapter,- });-- // Keep the evaluator contract unchanged while the model-facing schema- // resolves custom output instructions into the supported fields.- const modelFacingStructuredOutputSchema = z- .object({- scoreExplanation: params.structuredOutputSchema.shape.reasoning,- score: params.structuredOutputSchema.shape.score,- })- .describe(MODEL_FACING_OUTPUT_SCHEMA_DESCRIPTION);-- // llmaj egress: provider-bound messages (including base64-expanded inline- // media) plus schema, uncompressed.- const bytes =- Buffer.byteLength(- serializeProviderMessagesForEgress(providerMessages),- "utf8",- ) +- Buffer.byteLength(- serializeSchemaForEgress(modelFacingStructuredOutputSchema),- "utf8",- );-- const result = await generateLLMText({- ...llmParams,- messages: providerMessages,- traceInput: traceMessages,- output: createLLMOutput(modelFacingStructuredOutputSchema),- maxRetries: 1,+ ...params.modelConfig.modelParams,+ },+ messages: params.messages,+ outputResultSchema: params.structuredOutputSchema,trace: {targetProjectId: params.traceSinkParams.targetProjectId,traceId: params.traceSinkParams.traceId,@@ -311,17 +272,24 @@ export function createProductionEvalExecutionDeps(): EvalExecutionDeps {},});- // Record only after a successful send, like the other integrations.+ // llmaj egress: provider-bound messages (including base64-expanded inline+ // media) plus schema, uncompressed. Recorded only after a successful+ // send, like the other integrations.recordExportVolume({integration: "llmaj",- bytes,+ bytes:+ Buffer.byteLength(+ serializeProviderMessagesForEgress(result.providerMessages),+ "utf8",+ ) ++ Buffer.byteLength(+ serializeSchemaForEgress(result.modelFacingOutputSchema),+ "utf8",+ ),projectId: params.traceSinkParams.targetProjectId,});- return {- score: result.output.score,- reasoning: result.output.scoreExplanation,- };+ return result.output;},fetchModelConfig: async ({ projectId, provider, model, modelParams }) => {