Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(evals): send test evaluator runs through the live judge call (#18816)

Tobias Wochinger4e5adc805901
diff --git a/packages/shared/src/server/evals/llmEvaluatorExecution.test.ts b/packages/shared/src/server/evals/llmEvaluatorExecution.test.ts
index 12d6da991a..a31506653f 100644
--- a/packages/shared/src/server/evals/llmEvaluatorExecution.test.ts
+++ b/packages/shared/src/server/evals/llmEvaluatorExecution.test.ts
@@ -1,6 +1,100 @@
-import { describe, expect, it, vi } from "vitest";
-import { createNumericEvalOutputDefinition } from "../../features/evals/outputDefinition";
-import { executeLlmEvaluator } from "./llmEvaluatorExecution";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { z } from "zod";
+import {
+ compilePersistedEvalOutputDefinition,
+ createNumericEvalOutputDefinition,
+} from "../../features/evals/outputDefinition";
+import { ChatMessageRole, ChatMessageType, LLMAdapter } from "../llm/types";
+import {
+ executeLlmEvaluator,
+ generateLlmEvaluatorOutput,
+} from "./llmEvaluatorExecution";
+
+const { mockGenerateLLMText } = vi.hoisted(() => ({
+ mockGenerateLLMText: vi.fn(),
+}));
+
+vi.mock("../llm/llmText", async (importOriginal) => ({
+ ...(await importOriginal<typeof import("../llm/llmText")>()),
+ createLLMOutput: (schema: z.ZodType) => ({ schema }),
+ generateLLMText: mockGenerateLLMText,
+}));
+
+vi.mock("../llm/mediaMessages", () => ({
+ compileLangfuseMediaMessages: async ({
+ messages,
+ }: {
+ messages: Array<{ role: string; content: string }>;
+ }) => {
+ const mapped = messages.map(({ role, content }) => ({ role, content }));
+ return { providerMessages: mapped, traceMessages: mapped };
+ },
+}));
+
+describe("generateLlmEvaluatorOutput", () => {
+ beforeEach(() => {
+ mockGenerateLLMText.mockReset();
+ });
+
+ it("renames reasoning only for the model call and maps it back", async () => {
+ mockGenerateLLMText.mockResolvedValue({
+ output: { score: 0.8, scoreExplanation: "Good response" },
+ usage: { inputTokens: 10, outputTokens: 5 },
+ });
+ const { outputResultSchema } = compilePersistedEvalOutputDefinition(
+ createNumericEvalOutputDefinition({
+ scoreDescription: "score between 0 and 1",
+ reasoningDescription: "why this score was given",
+ }),
+ );
+
+ const result = await generateLlmEvaluatorOutput({
+ projectId: "project-123",
+ connection: { secretKey: "secret" },
+ modelParams: {
+ provider: "openai",
+ model: "gpt-4.1",
+ adapter: LLMAdapter.OpenAI,
+ },
+ messages: [
+ {
+ role: ChatMessageRole.User,
+ type: ChatMessageType.User,
+ content: "Judge this answer",
+ },
+ ],
+ outputResultSchema,
+ trace: {
+ targetProjectId: "project-123",
+ traceId: "trace-123",
+ traceName: "Judge trace",
+ environment: "langfuse-llm-as-a-judge",
+ },
+ });
+
+ const call = mockGenerateLLMText.mock.calls[0][0];
+ expect(call).toMatchObject({
+ maxRetries: 1,
+ trace: { traceId: "trace-123" },
+ });
+ const modelSchema = z.toJSONSchema(call.output.schema as z.ZodType);
+ expect(modelSchema).toMatchObject({
+ description:
+ 'Return only top-level "score" and "scoreExplanation". Put other requested fields inside "scoreExplanation".',
+ properties: {
+ scoreExplanation: {
+ type: "string",
+ description: "why this score was given",
+ },
+ score: { type: "number", description: "score between 0 and 1" },
+ },
+ required: ["scoreExplanation", "score"],
+ });
+ expect(modelSchema.properties).not.toHaveProperty("reasoning");
+ expect(result.output).toEqual({ score: 0.8, reasoning: "Good response" });
+ expect(result.usage).toEqual({ inputTokens: 10, outputTokens: 5 });
+ });
+});
describe("executeLlmEvaluator", () => {
it("interpolates a single user prompt message", async () => {
diff --git a/packages/shared/src/server/evals/llmEvaluatorExecution.ts b/packages/shared/src/server/evals/llmEvaluatorExecution.ts
index baddabffdf..c388bd1f1f 100644
--- a/packages/shared/src/server/evals/llmEvaluatorExecution.ts
+++ b/packages/shared/src/server/evals/llmEvaluatorExecution.ts
@@ -14,7 +14,20 @@ import {
ChatMessageRole,
ChatMessageType,
type ChatMessage,
+ type ModelParams,
+ type TraceSinkParams,
} from "../llm/types";
+import {
+ createLLMOutput,
+ generateLLMText,
+ mapLegacyLLMCompletionParams,
+ type LLMConnection,
+} from "../llm/llmText";
+import { compileLangfuseMediaMessages } from "../llm/mediaMessages";
+import { z } from "zod";
+
+const MODEL_FACING_OUTPUT_SCHEMA_DESCRIPTION =
+ 'Return only top-level "score" and "scoreExplanation". Put other requested fields inside "scoreExplanation".';
const CHAT_MESSAGE_BUILDERS = {
system: (content: string) => ({
@@ -56,6 +69,60 @@ function buildEvalMessages(params: {
};
}
+/**
+ * The single model call behind live and test LLM-as-a-judge runs, so a test
+ * run sends the provider exactly what a live run sends.
+ *
+ * The model sees `scoreExplanation` instead of `reasoning`: thinking-capable
+ * models treat a `reasoning` field as their scratchpad and drop the score.
+ * The output is mapped back to the `{ score, reasoning }` evaluator contract.
+ */
+export async function generateLlmEvaluatorOutput(params: {
+ projectId: string;
+ connection: LLMConnection;
+ modelParams: ModelParams;
+ messages: ChatMessage[];
+ outputResultSchema: z.ZodObject<{ reasoning: z.ZodString; score: z.ZodType }>;
+ trace: TraceSinkParams;
+}) {
+ const llmParams = mapLegacyLLMCompletionParams({
+ connection: params.connection,
+ messages: params.messages,
+ modelParams: params.modelParams,
+ });
+ const { providerMessages, traceMessages } =
+ await compileLangfuseMediaMessages({
+ projectId: params.projectId,
+ messages: params.messages,
+ adapter: params.modelParams.adapter,
+ });
+ const modelFacingOutputSchema = z
+ .object({
+ scoreExplanation: params.outputResultSchema.shape.reasoning,
+ score: params.outputResultSchema.shape.score,
+ })
+ .describe(MODEL_FACING_OUTPUT_SCHEMA_DESCRIPTION);
+
+ const result = await generateLLMText({
+ ...llmParams,
+ messages: providerMessages,
+ traceInput: traceMessages,
+ output: createLLMOutput(modelFacingOutputSchema),
+ maxRetries: 1,
+ trace: params.trace,
+ });
+
+ return {
+ output: {
+ score: result.output.score,
+ reasoning: result.output.scoreExplanation,
+ },
+ usage: result.usage,
+ providerMessages,
+ modelFacingOutputSchema,
+ };
+}
+
export async function executeLlmEvaluator(params: {
promptMessages: PersistedEvaluatorPromptMessages;
variables: ExtractedVariable[];
diff --git a/web/src/__tests__/server/unit/evaluator-v2-test-evaluator.servertest.ts b/web/src/__tests__/server/unit/evaluator-v2-test-evaluator.servertest.ts
index b2e171da63..b5d97fd4be 100644
--- a/web/src/__tests__/server/unit/evaluator-v2-test-evaluator.servertest.ts
+++ b/web/src/__tests__/server/unit/evaluator-v2-test-evaluator.servertest.ts
@@ -3,10 +3,9 @@ import {
LLMAdapter,
} from "@langfuse/shared";
import {
- compileLangfuseMediaMessages,
DefaultEvalModelService,
findModel,
- generateLLMText,
+ generateLlmEvaluatorOutput,
resolveConfiguredCodeEvalDispatcher,
runCodeBasedEvaluationDispatch,
} from "@langfuse/shared/src/server";
@@ -23,8 +22,7 @@ vi.mock("@langfuse/shared/src/server", async () => ({
DefaultEvalModelService: {
fetchValidModelConfig: vi.fn(),
},
- compileLangfuseMediaMessages: vi.fn(),
- generateLLMText: vi.fn(),
+ generateLlmEvaluatorOutput: vi.fn(),
findModel: vi.fn(),
resolveConfiguredCodeEvalDispatcher: vi.fn(),
runCodeBasedEvaluationDispatch: vi.fn(),
@@ -56,13 +54,7 @@ describe("testEvaluator", () => {
},
} as unknown as ValidModelConfig,
});
- vi.mocked(compileLangfuseMediaMessages).mockImplementation(
- async ({ messages }) => {
- const mapped = messages.map(({ role, content }) => ({ role, content }));
- return { providerMessages: mapped, traceMessages: mapped } as never;
- },
- );
- vi.mocked(generateLLMText).mockResolvedValue({
+ vi.mocked(generateLlmEvaluatorOutput).mockResolvedValue({
output: { score: 1, reasoning: "Correct" },
usage: { inputTokens: 100, outputTokens: 50, totalTokens: 150 },
} as never);
@@ -139,12 +131,10 @@ describe("testEvaluator", () => {
startTime,
shouldReadFromObservationsTable: true,
});
- expect(compileLangfuseMediaMessages).toHaveBeenCalledWith(
- expect.objectContaining({ projectId: "project-1" }),
- );
- expect(generateLLMText).toHaveBeenCalledWith(
+ // Live runs send the model this same call, so Test reproduces them.
+ expect(generateLlmEvaluatorOutput).toHaveBeenCalledWith(
expect.objectContaining({
- maxRetries: 1,
+ projectId: "project-1",
messages: [
expect.objectContaining({
role: "system",
@@ -155,16 +145,12 @@ describe("testEvaluator", () => {
content: "Judge It arrives tomorrow.",
}),
],
- traceInput: [
- expect.objectContaining({
- role: "system",
- content: "Judge consistently",
+ outputResultSchema: expect.objectContaining({
+ shape: expect.objectContaining({
+ reasoning: expect.anything(),
+ score: expect.anything(),
}),
- expect.objectContaining({
- role: "user",
- content: "Judge It arrives tomorrow.",
- }),
- ],
+ }),
trace: expect.objectContaining({
evaluationContext: {
evaluatorId: "evaluator-1",
diff --git a/web/src/features/evals/v2/server/evaluators/testEvaluator.ts b/web/src/features/evals/v2/server/evaluators/testEvaluator.ts
index 3f8ac76f10..7a8f87cd96 100644
--- a/web/src/features/evals/v2/server/evaluators/testEvaluator.ts
+++ b/web/src/features/evals/v2/server/evaluators/testEvaluator.ts
@@ -5,22 +5,19 @@ import {
import { decrypt } from "@langfuse/shared/encryption";
import {
buildEvalExecutionData,
- compileLangfuseMediaMessages,
createDecisionModelClient,
createW3CTraceId,
decryptAndParseExtraHeaders,
DefaultEvalModelService,
- createLLMOutput,
executeDecisionModelEvaluator,
executeLlmEvaluator,
extractObservationVariables,
findModel,
- generateLLMText,
+ generateLlmEvaluatorOutput,
isAllowedDecisionModel,
LLMAdapter,
OPENAI_DECISION_MODEL_IDS,
LangfuseInternalTraceEnvironment,
- mapLegacyLLMCompletionParams,
matchPricingTier,
resolveConfiguredCodeEvalDispatcher,
runCodeBasedEvaluationDispatch,
@@ -236,29 +233,17 @@ async function testLlmEvaluator(params: {
interpolatedPrompt: prompt,
}) => {
interpolatedPrompt = prompt;
- const modelParams = {
- provider: modelConfig.config.provider,
- model: modelConfig.config.model,
- adapter: modelConfig.config.apiKey.adapter,
- ...modelConfig.config.modelParams,
- };
- const llmParams = mapLegacyLLMCompletionParams({
+ const result = await generateLlmEvaluatorOutput({
+ projectId: params.projectId,
connection: modelConfig.config.apiKey,
- messages,
- modelParams,
- });
- const { providerMessages, traceMessages } =
- await compileLangfuseMediaMessages({
- projectId: params.projectId,
- messages,
+ modelParams: {
+ provider: modelConfig.config.provider,
+ model: modelConfig.config.model,
adapter: modelConfig.config.apiKey.adapter,
- });
- const result = await generateLLMText({
- ...llmParams,
- messages: providerMessages,
- traceInput: traceMessages,
- output: createLLMOutput(compiledOutputDefinition.outputResultSchema),
- maxRetries: 1,
+ ...modelConfig.config.modelParams,
+ },
+ messages,
+ outputResultSchema: compiledOutputDefinition.outputResultSchema,
trace: {
targetProjectId: params.projectId,
traceId: executionTraceId,
diff --git a/worker/src/__tests__/evalService.test.ts b/worker/src/__tests__/evalService.test.ts
index 143101dafc..67a52e62fb 100644
--- a/worker/src/__tests__/evalService.test.ts
+++ b/worker/src/__tests__/evalService.test.ts
@@ -33,19 +33,21 @@ import {
} from "../features/evaluation/evalService";
import { requiresDatabaseLookup } from "../features/evaluation/traceFilterUtils";
-// Mock the shared LLM runtime with default passthrough behavior.
+// Mock the shared evaluator model call with default passthrough behavior.
vi.mock("@langfuse/shared/src/server", async () => {
- const actual = await vi.importActual("@langfuse/shared/src/server");
+ const actual = await vi.importActual<
+ typeof import("@langfuse/shared/src/server")
+ >("@langfuse/shared/src/server");
return {
...actual,
- generateLLMText: vi.fn().mockImplementation(actual.generateLLMText as any),
+ generateLlmEvaluatorOutput: vi.fn(actual.generateLlmEvaluatorOutput),
};
});
// Import the mocked function
import {
EvalExecutionQueue,
- generateLLMText,
+ generateLlmEvaluatorOutput,
} from "@langfuse/shared/src/server";
import { UnrecoverableError } from "../errors/UnrecoverableError";
@@ -2539,8 +2541,10 @@ Respond with JSON: {"score": <number>, "reasoning": "<explanation>"}`;
test("handles LLM timeout gracefully", async () => {
const { projectId } = await createOrgProjectAndApiKey();
// Set up the mock to simulate timeout for this test only
- const mockGenerateLLMText = vi.mocked(generateLLMText);
- mockGenerateLLMText.mockRejectedValueOnce(
+ const mockGenerateLlmEvaluatorOutput = vi.mocked(
+ generateLlmEvaluatorOutput,
+ );
+ mockGenerateLlmEvaluatorOutput.mockRejectedValueOnce(
new ApiError("Request timeout after 120000ms", 500),
);
@@ -2644,7 +2648,7 @@ Respond with JSON: {"score": <number>, "reasoning": "<explanation>"}`;
expect(jobs[0].status.toString()).toBe("PENDING");
// Clean up the mock after this test
- mockGenerateLLMText.mockReset();
+ mockGenerateLlmEvaluatorOutput.mockReset();
}, 15_000);
});
@@ -3446,12 +3450,15 @@ Respond with JSON: {"score": <number>, "reasoning": "<explanation>"}`;
// Mock the LLM runtime to capture the trace parameters.
let capturedTraceSinkParams: any = null;
- vi.mocked(generateLLMText).mockImplementationOnce((async (
+ vi.mocked(generateLlmEvaluatorOutput).mockImplementationOnce((async (
params: any,
) => {
capturedTraceSinkParams = params.trace;
return {
- output: { score: 0.8, scoreExplanation: "Good response" },
+ output: { score: 0.8, reasoning: "Good response" },
+ usage: {},
+ providerMessages: [],
+ modelFacingOutputSchema: params.outputResultSchema,
};
}) as any);
@@ -3487,11 +3494,14 @@ Respond with JSON: {"score": <number>, "reasoning": "<explanation>"}`;
const jobExecutionId = randomUUID();
const templateId = randomUUID();
- vi.mocked(generateLLMText).mockResolvedValueOnce({
+ vi.mocked(generateLlmEvaluatorOutput).mockResolvedValueOnce({
output: {
score: 0,
- scoreExplanation: "The response is safe and not toxic.",
+ reasoning: "The response is safe and not toxic.",
},
+ usage: {},
+ providerMessages: [],
+ modelFacingOutputSchema: {},
} as any);
await upsertTrace({
diff --git a/worker/src/features/evaluation/__tests__/evalExecutionDeps.test.ts b/worker/src/features/evaluation/__tests__/evalExecutionDeps.test.ts
index 4f0c05e759..3c23f492a9 100644
--- a/worker/src/features/evaluation/__tests__/evalExecutionDeps.test.ts
+++ b/worker/src/features/evaluation/__tests__/evalExecutionDeps.test.ts
@@ -3,26 +3,19 @@ import { z } from "zod";
import { createProductionEvalExecutionDeps } from "../evalExecutionDeps";
import { EXPORT_VOLUME_METRIC } from "../../../services/exportVolumeMetric";
-const {
- mockCompileLangfuseMediaMessages,
- mockCreateLLMOutput,
- mockGenerateLLMText,
- mockRecordIncrement,
-} = vi.hoisted(() => ({
- mockCompileLangfuseMediaMessages: vi.fn(),
- mockCreateLLMOutput: vi.fn(),
- mockGenerateLLMText: vi.fn(),
- mockRecordIncrement: vi.fn(),
-}));
+const { mockGenerateLlmEvaluatorOutput, mockRecordIncrement } = vi.hoisted(
+ () => ({
+ mockGenerateLlmEvaluatorOutput: vi.fn(),
+ mockRecordIncrement: vi.fn(),
+ }),
+);
vi.mock("@langfuse/shared/src/server", async (importOriginal) => {
const original =
await importOriginal<typeof import("@langfuse/shared/src/server")>();
return {
...original,
- compileLangfuseMediaMessages: mockCompileLangfuseMediaMessages,
- createLLMOutput: mockCreateLLMOutput,
- generateLLMText: mockGenerateLLMText,
+ generateLlmEvaluatorOutput: mockGenerateLlmEvaluatorOutput,
recordIncrement: mockRecordIncrement,
};
});
@@ -38,64 +31,52 @@ vi.mock("../../../env", async (importOriginal) => {
};
});
+const modelConfig = {
+ provider: "openai",
+ model: "gpt-4.1",
+ apiKey: { adapter: "openai", secretKey: "secret" },
+ adapter: "openai" as any,
+ modelParams: {},
+};
+
+const traceSinkParams = {
+ targetProjectId: "project-123",
+ traceId: "trace-123",
+ traceName: "Judge trace",
+ environment: "langfuse-llm-as-a-judge",
+ metadata: { score_id: "score-123" },
+};
+
+const structuredOutputSchema = z.object({
+ reasoning: z.string().describe("why this score was given"),
+ score: z.number().describe("score between 0 and 1"),
+});
+
describe("createProductionEvalExecutionDeps", () => {
beforeEach(() => {
vi.clearAllMocks();
- mockCompileLangfuseMediaMessages.mockImplementation(
- async ({ messages }) => {
- const mapped = messages.map(({ role, content }: any) => ({
- role,
- content,
- }));
- return { providerMessages: mapped, traceMessages: mapped };
- },
- );
- mockCreateLLMOutput.mockImplementation((schema) => ({ schema }));
- mockGenerateLLMText.mockResolvedValue({ output: { completion: "ok" } });
+ mockGenerateLlmEvaluatorOutput.mockResolvedValue({
+ output: { score: 0.8, reasoning: "Good response" },
+ usage: {},
+ providerMessages: [],
+ modelFacingOutputSchema: structuredOutputSchema,
+ });
});
- it("enables internal direct event write for llm-as-a-judge traces", async () => {
+ it("calls the shared evaluator model call with direct event writes", async () => {
const deps = createProductionEvalExecutionDeps();
- await deps.callLLM({
- messages: [
- {
- role: "user",
- type: "user",
- content: "Judge this answer",
- },
- ],
- modelConfig: {
- provider: "openai",
- model: "gpt-4.1",
- apiKey: {
- adapter: "openai",
- secretKey: "secret",
- },
- adapter: "openai" as any,
- modelParams: {},
- },
- structuredOutputSchema: z.object({
- reasoning: z.string(),
- score: z.number(),
- }),
- traceSinkParams: {
- targetProjectId: "project-123",
- traceId: "trace-123",
- traceName: "Judge trace",
- environment: "langfuse-llm-as-a-judge",
- metadata: {
- score_id: "score-123",
- },
- },
- });
+ const result = await deps.callLLM({
+ messages: [{ role: "user", type: "user", content: "Judge this answer" }],
+ modelConfig,
+ structuredOutputSchema,
+ traceSinkParams,
+ } as any);
- expect(mockCompileLangfuseMediaMessages).toHaveBeenCalledWith(
- expect.objectContaining({ projectId: "project-123" }),
- );
- expect(mockGenerateLLMText).toHaveBeenCalledWith(
+ expect(mockGenerateLlmEvaluatorOutput).toHaveBeenCalledWith(
expect.objectContaining({
- maxRetries: 1,
+ projectId: "project-123",
+ outputResultSchema: structuredOutputSchema,
trace: expect.objectContaining({
traceId: "trace-123",
environment: "langfuse-llm-as-a-judge",
@@ -105,73 +86,10 @@ describe("createProductionEvalExecutionDeps", () => {
}),
}),
);
- });
-
- it("renames reasoning only for the model call and maps it back", async () => {
- mockGenerateLLMText.mockResolvedValue({
- output: { score: 0.8, scoreExplanation: "Good response" },
- });
- const deps = createProductionEvalExecutionDeps();
- const structuredOutputSchema = z.object({
- reasoning: z.string().describe("why this score was given"),
- score: z.number().describe("score between 0 and 1"),
- });
-
- const result = await deps.callLLM({
- messages: [
- {
- role: "user",
- type: "user",
- content: "Judge this answer",
- },
- ],
- modelConfig: {
- provider: "openai",
- model: "gpt-4.1",
- apiKey: { adapter: "openai", secretKey: "secret" },
- adapter: "openai" as any,
- modelParams: {},
- },
- structuredOutputSchema,
- traceSinkParams: {
- targetProjectId: "project-123",
- traceId: "trace-123",
- traceName: "Judge trace",
- environment: "langfuse-llm-as-a-judge",
- metadata: {},
- },
- });
-
- const modelOutput = mockGenerateLLMText.mock.calls[0][0].output as {
- schema: z.ZodType;
- };
- expect(z.toJSONSchema(modelOutput.schema)).toMatchObject({
- description:
- 'Return only top-level "score" and "scoreExplanation". Put other requested fields inside "scoreExplanation".',
- properties: {
- scoreExplanation: { type: "string" },
- score: { type: "number" },
- },
- required: ["scoreExplanation", "score"],
- });
- expect(z.toJSONSchema(modelOutput.schema).properties).not.toHaveProperty(
- "reasoning",
- );
- expect(
- structuredOutputSchema.safeParse({
- score: 0.8,
- reasoning: "Good response",
- }).success,
- ).toBe(true);
expect(result).toEqual({ score: 0.8, reasoning: "Good response" });
});
it("records llmaj export volume using the schema's JSON Schema form", async () => {
- const deps = createProductionEvalExecutionDeps();
-
- const messages = [
- { role: "user", type: "user", content: "Judge this answer" },
- ];
const providerMessages = [
{
role: "user",
@@ -185,34 +103,27 @@ describe("createProductionEvalExecutionDeps", () => {
],
},
];
- mockCompileLangfuseMediaMessages.mockResolvedValueOnce({
- providerMessages,
- traceMessages: messages,
- });
// Production passes a Zod schema, not a plain object.
- const structuredOutputSchema = z.object({
- reasoning: z.string().describe("why this score was given"),
- score: z.number().describe("score between 0 and 1"),
+ const modelFacingSchema = z
+ .object({
+ scoreExplanation: structuredOutputSchema.shape.reasoning,
+ score: structuredOutputSchema.shape.score,
+ })
+ .describe("model-facing schema");
+ mockGenerateLlmEvaluatorOutput.mockResolvedValueOnce({
+ output: { score: 0.8, reasoning: "Good response" },
+ usage: {},
+ providerMessages,
+ modelFacingOutputSchema: modelFacingSchema,
});
+ const deps = createProductionEvalExecutionDeps();
await deps.callLLM({
- messages: messages as any,
- modelConfig: {
- provider: "openai",
- model: "gpt-4.1",
- apiKey: { adapter: "openai", secretKey: "secret" },
- adapter: "openai" as any,
- modelParams: {},
- },
- structuredOutputSchema: structuredOutputSchema as any,
- traceSinkParams: {
- targetProjectId: "project-123",
- traceId: "trace-123",
- traceName: "Judge trace",
- environment: "langfuse-llm-as-a-judge",
- metadata: {},
- },
- });
+ messages: [{ role: "user", type: "user", content: "Judge this answer" }],
+ modelConfig,
+ structuredOutputSchema,
+ traceSinkParams,
+ } as any);
const serializedProviderMessages = JSON.stringify(
providerMessages,
@@ -221,14 +132,6 @@ describe("createProductionEvalExecutionDeps", () => {
? Buffer.from(value).toString("base64")
: value,
);
- const modelFacingSchema = z
- .object({
- scoreExplanation: structuredOutputSchema.shape.reasoning,
- score: structuredOutputSchema.shape.score,
- })
- .describe(
- 'Return only top-level "score" and "scoreExplanation". Put other requested fields inside "scoreExplanation".',
- );
const expectedBytes =
Buffer.byteLength(serializedProviderMessages, "utf8") +
Buffer.byteLength(
@@ -236,19 +139,32 @@ describe("createProductionEvalExecutionDeps", () => {
"utf8",
);
- expect(expectedBytes).toBeGreaterThan(0);
expect(mockRecordIncrement).toHaveBeenCalledWith(
EXPORT_VOLUME_METRIC,
expectedBytes,
{ integration: "llmaj" },
);
// Not the Zod _def form.
- const zodDefBytes = Buffer.byteLength(
- JSON.stringify(modelFacingSchema),
- "utf8",
- );
expect(expectedBytes).not.toBe(
- Buffer.byteLength(JSON.stringify(messages), "utf8") + zodDefBytes,
+ Buffer.byteLength(serializedProviderMessages, "utf8") +
+ Buffer.byteLength(JSON.stringify(modelFacingSchema), "utf8"),
);
});
+
+ it("does not record export volume when the model call fails", async () => {
+ mockGenerateLlmEvaluatorOutput.mockRejectedValueOnce(new Error("boom"));
+ const deps = createProductionEvalExecutionDeps();
+
+ await expect(
+ deps.callLLM({
+ messages: [
+ { role: "user", type: "user", content: "Judge this answer" },
+ ],
+ modelConfig,
+ structuredOutputSchema,
+ traceSinkParams,
+ } as any),
+ ).rejects.toThrow("boom");
+ expect(mockRecordIncrement).not.toHaveBeenCalled();
+ });
});
diff --git a/worker/src/features/evaluation/evalExecutionDeps.ts b/worker/src/features/evaluation/evalExecutionDeps.ts
index 6c20c134f6..4ef05e6653 100644
--- a/worker/src/features/evaluation/evalExecutionDeps.ts
+++ b/worker/src/features/evaluation/evalExecutionDeps.ts
@@ -6,12 +6,10 @@ import { prisma } from "@langfuse/shared/src/db";
import { decrypt } from "@langfuse/shared/encryption";
import {
buildEventBucketPrefix,
- compileLangfuseMediaMessages,
- createLLMOutput,
createDecisionModelClient,
decryptAndParseExtraHeaders,
DefaultEvalModelService,
- generateLLMText,
+ generateLlmEvaluatorOutput,
IngestionQueue,
LLMAdapter,
mapLegacyLLMCompletionParams,
@@ -34,9 +32,6 @@ type StructuredOutputSchema = z.ZodObject<{
score: z.ZodType;
}>;
-const MODEL_FACING_OUTPUT_SCHEMA_DESCRIPTION =
- 'Return only top-level "score" and "scoreExplanation". Put other requested fields inside "scoreExplanation".';
-
/**
* Result of fetching model configuration.
*/
@@ -255,51 +250,17 @@ export function createProductionEvalExecutionDeps(): EvalExecutionDeps {
typeof mapLegacyLLMCompletionParams
>[0]["modelParams"]["adapter"];
- const modelParams = {
- provider: params.modelConfig.provider,
- model: params.modelConfig.model,
- adapter,
- ...params.modelConfig.modelParams,
- };
- const llmParams = mapLegacyLLMCompletionParams({
+ const result = await generateLlmEvaluatorOutput({
+ projectId: params.traceSinkParams.targetProjectId,
connection,
- messages: params.messages,
- modelParams,
- });
- const { providerMessages, traceMessages } =
- await compileLangfuseMediaMessages({
- projectId: params.traceSinkParams.targetProjectId,
- messages: params.messages,
+ modelParams: {
+ provider: params.modelConfig.provider,
+ model: params.modelConfig.model,
adapter,
- });
-
- // Keep the evaluator contract unchanged while the model-facing schema
- // resolves custom output instructions into the supported fields.
- const modelFacingStructuredOutputSchema = z
- .object({
- scoreExplanation: params.structuredOutputSchema.shape.reasoning,
- score: params.structuredOutputSchema.shape.score,
- })
- .describe(MODEL_FACING_OUTPUT_SCHEMA_DESCRIPTION);
-
- // llmaj egress: provider-bound messages (including base64-expanded inline
- // media) plus schema, uncompressed.
- const bytes =
- Buffer.byteLength(
- serializeProviderMessagesForEgress(providerMessages),
- "utf8",
- ) +
- Buffer.byteLength(
- serializeSchemaForEgress(modelFacingStructuredOutputSchema),
- "utf8",
- );
-
- const result = await generateLLMText({
- ...llmParams,
- messages: providerMessages,
- traceInput: traceMessages,
- output: createLLMOutput(modelFacingStructuredOutputSchema),
- maxRetries: 1,
+ ...params.modelConfig.modelParams,
+ },
+ messages: params.messages,
+ outputResultSchema: params.structuredOutputSchema,
trace: {
targetProjectId: params.traceSinkParams.targetProjectId,
traceId: params.traceSinkParams.traceId,
@@ -311,17 +272,24 @@ export function createProductionEvalExecutionDeps(): EvalExecutionDeps {
},
});
- // Record only after a successful send, like the other integrations.
+ // llmaj egress: provider-bound messages (including base64-expanded inline
+ // media) plus schema, uncompressed. Recorded only after a successful
+ // send, like the other integrations.
recordExportVolume({
integration: "llmaj",
- bytes,
+ bytes:
+ Buffer.byteLength(
+ serializeProviderMessagesForEgress(result.providerMessages),
+ "utf8",
+ ) +
+ Buffer.byteLength(
+ serializeSchemaForEgress(result.modelFacingOutputSchema),
+ "utf8",
+ ),
projectId: params.traceSinkParams.targetProjectId,
});
- return {
- score: result.output.score,
- reasoning: result.output.scoreExplanation,
- };
+ return result.output;
},
fetchModelConfig: async ({ projectId, provider, model, modelParams }) => {