🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
fix(evals): attribute terminal LLM-as-judge failures by reason (#18821)
Tobias Wochinger0793320d5f4b
diff --git a/worker/src/__tests__/evalService.test.ts b/worker/src/__tests__/evalService.test.tsindex 67a52e62fb..76b5d2e46b 100644--- a/worker/src/__tests__/evalService.test.ts+++ b/worker/src/__tests__/evalService.test.ts@@ -50,6 +50,10 @@ import {generateLlmEvaluatorOutput,} from "@langfuse/shared/src/server";import { UnrecoverableError } from "../errors/UnrecoverableError";+import {+ EvalFailureReason,+ UnrecoverableEvalFailureError,+} from "../features/evaluation/evalFailureErrors";let OPENAI_API_KEY = process.env.OPENAI_API_KEY;// Check for both OPENAI_API_KEY and LANGFUSE_LLM_CONNECTION_OPENAI_KEY@@ -2136,8 +2140,9 @@ Respond with JSON: {"score": <number>, "reasoning": "<explanation>"}`;};await expect(evaluate({ event: payload })).rejects.toThrowError(- new UnrecoverableError(+ new UnrecoverableEvalFailureError(`Invalid model configuration for job ${jobExecutionId}: API key for provider "openai" not found in project ${projectId}`,+ EvalFailureReason.InvalidModelConfig,),);diff --git a/worker/src/features/evaluation/decisionModel/runDecisionModelEvaluation.ts b/worker/src/features/evaluation/decisionModel/runDecisionModelEvaluation.tsindex bc27764b1a..3fdab0906b 100644--- a/worker/src/features/evaluation/decisionModel/runDecisionModelEvaluation.ts+++ b/worker/src/features/evaluation/decisionModel/runDecisionModelEvaluation.ts@@ -25,6 +25,10 @@ import {type ExtractedVariable,} from "@langfuse/shared/src/server";import { UnrecoverableError } from "../../../errors/UnrecoverableError";+import {+ EvalFailureReason,+ UnrecoverableEvalFailureError,+} from "../evalFailureErrors";import { createW3CTraceId } from "../../utils";import { type EvalExecutionResult } from "../evalCompletion";import { type EvalExecutionDeps } from "../evalExecutionDeps";@@ -145,8 +149,9 @@ export async function runDecisionModelEvaluation({logger.warn(`Eval job ${jobExecutionId} will fail. ${modelConfigError}`,);- throw new UnrecoverableError(+ throw new UnrecoverableEvalFailureError(`Invalid model configuration for job ${jobExecutionId}: ${modelConfigError}`,+ EvalFailureReason.InvalidModelConfig,);}@@ -180,7 +185,10 @@ export async function runDecisionModelEvaluation({} catch (e) {if (e instanceof DecisionModelEvaluatorError) {span.setAttribute("eval.execution.outcome", "invalid_model_output");- throw new UnrecoverableError(e.message);+ throw new UnrecoverableEvalFailureError(+ e.message,+ EvalFailureReason.InvalidModelOutput,+ );}const classification = classifyEvaluatorLlmError(e);diff --git a/worker/src/features/evaluation/evalCompletion.ts b/worker/src/features/evaluation/evalCompletion.tsindex 12aadde966..7ca48cc2ba 100644--- a/worker/src/features/evaluation/evalCompletion.ts+++ b/worker/src/features/evaluation/evalCompletion.ts@@ -7,6 +7,7 @@ import {} from "@langfuse/shared/src/server";import { buildEvalScoreWritePayloads } from "./evalScoreEvent";import { type EvalExecutionDeps } from "./evalExecutionDeps";+import { EvalFailureError, EvalFailureReason } from "./evalFailureErrors";import {recordEvalScorePersistenceOutcome,type ObservationEvalExecutionType,@@ -85,8 +86,9 @@ export async function completeEvalExecution({);logger.error(`Failed to persist score: ${e}`, e);traceException(e);- throw new Error(+ throw new EvalFailureError(`Failed to write score ${jobOutputScoreId} into IngestionQueue`,+ EvalFailureReason.ScorePersistence,);}diff --git a/worker/src/features/evaluation/evalExecutionMetrics.test.ts b/worker/src/features/evaluation/evalExecutionMetrics.test.tsindex c6df2cf255..90ce2564f5 100644--- a/worker/src/features/evaluation/evalExecutionMetrics.test.ts+++ b/worker/src/features/evaluation/evalExecutionMetrics.test.ts@@ -10,12 +10,13 @@ import { beforeEach, describe, expect, it, vi } from "vitest";import {__test,getCodeEvalTerminalErrorOutcome,+ getLlmEvalTerminalErrorAttribution,getLlmEvalTerminalErrorOutcome,recordEvalScorePersistenceOutcome,recordEvalTerminalOutcome,recordEvalTimeToFirstAttempt,} from "./evalExecutionMetrics";-+import { EvalFailureError, EvalFailureReason } from "./evalFailureErrors";vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({...(await importOriginal<typeof import("@langfuse/shared/src/server")>()),recordDistribution: vi.fn(),@@ -139,6 +140,45 @@ describe("evaluation execution metrics", () => {},);+ it("attributes classified LLM platform errors by error kind", () => {+ const error = new Error("unexpected AI SDK failure");++ expect(+ getLlmEvalTerminalErrorAttribution({+ error,+ classification: {+ kind: "ai-sdk",+ message: error.message,+ isRetryable: false,+ error,+ blockReason: null,+ },+ isUnrecoverable: false,+ }),+ ).toEqual({ outcome: "platform_error", reason: "llm_ai_sdk" });+ });++ it("prefers an explicit failure reason over LLM classification", () => {+ const error = new EvalFailureError(+ "Failed to download observation: DNS lookup failed",+ EvalFailureReason.ObservationDownload,+ );++ expect(+ getLlmEvalTerminalErrorAttribution({+ error,+ classification: {+ kind: "evaluator-policy",+ message: error.message,+ isRetryable: false,+ error,+ blockReason: null,+ },+ isUnrecoverable: false,+ }),+ ).toEqual({ outcome: "platform_error", reason: "observation_download" });+ });+it.each([{code: CodeEvalDispatcherErrorCodes.USER_CODE_ERROR,diff --git a/worker/src/features/evaluation/evalExecutionMetrics.ts b/worker/src/features/evaluation/evalExecutionMetrics.tsindex 8a7cbcc7a1..fc39ee89e9 100644--- a/worker/src/features/evaluation/evalExecutionMetrics.ts+++ b/worker/src/features/evaluation/evalExecutionMetrics.ts@@ -6,6 +6,7 @@ import {recordIncrement,type EvaluatorLlmErrorClassification,} from "@langfuse/shared/src/server";+import { EvalFailureReason, getEvalFailureReason } from "./evalFailureErrors";export type ObservationEvalExecutionType =| typeof EvalTemplateType.LLM_AS_JUDGE@@ -18,6 +19,21 @@ export type EvalExecutionTerminalOutcome =| "customer_error"| "cancelled";+const CUSTOMER_EVAL_FAILURE_REASONS: ReadonlySet<EvalFailureReason> =+ new Set<CustomerEvalFailureReason>([+ EvalFailureReason.InvalidModelConfig,+ EvalFailureReason.InvalidModelOutput,+ ]);++const LLM_ERROR_KIND_REASONS = {+ provider: "llm_provider",+ validation: "llm_validation",+ "ai-sdk": "llm_ai_sdk",+ timeout: "llm_timeout",+ abort: "llm_abort",+ "evaluator-policy": "llm_evaluator_policy",+} as const satisfies Record<EvaluatorLlmErrorClassification["kind"], string>;+type EvalScoreCountBucket = "0" | "1" | "2" | "3-5" | "6-10" | "11+";const LLM_EVAL_CUSTOMER_OUTPUT_ERROR_NAMES: ReadonlySet<string> = new Set([@@ -53,13 +69,49 @@ export function recordEvalTimeToFirstAttempt(export function recordEvalTerminalOutcome(executionType: ObservationEvalExecutionType,outcome: EvalExecutionTerminalOutcome,+ platformErrorReason?: EvalPlatformErrorReason,): void {recordIncrement("langfuse.evaluation.execution.terminal", 1, {evaluator_type: getEvaluatorTypeTag(executionType),outcome,+ ...(outcome === "platform_error" && platformErrorReason+ ? { reason: platformErrorReason }+ : {}),});}+/**+ * Attributes a terminal LLM evaluator failure. Known failure sites carry an+ * explicit reason; everything else falls back to the LLM classification, then+ * to whether the error was raised as unrecoverable.+ */+export function getLlmEvalTerminalErrorAttribution(params: {+ error: unknown;+ classification: EvaluatorLlmErrorClassification | null;+ isUnrecoverable: boolean;+}): LlmEvalTerminalErrorAttribution {+ const { error, classification, isUnrecoverable } = params;++ const failureReason = getEvalFailureReason(error);+ if (failureReason) {+ return isCustomerEvalFailureReason(failureReason)+ ? { outcome: "customer_error" }+ : { outcome: "platform_error", reason: failureReason };+ }++ if (classification) {+ const outcome = getLlmEvalTerminalErrorOutcome(classification);+ return outcome === "platform_error"+ ? { outcome, reason: LLM_ERROR_KIND_REASONS[classification.kind] }+ : { outcome };+ }++ return {+ outcome: "platform_error",+ reason: isUnrecoverable ? "unrecoverable" : "unexpected",+ };+}+export function recordEvalScorePersistenceOutcome(executionType: ObservationEvalExecutionType,outcome: "success" | "failure",@@ -163,6 +215,26 @@ function getEvalScoreCountBucket(scoreCount: number): EvalScoreCountBucket {return "11+";}+function isCustomerEvalFailureReason(+ reason: EvalFailureReason,+): reason is CustomerEvalFailureReason {+ return CUSTOMER_EVAL_FAILURE_REASONS.has(reason);+}++type EvalPlatformErrorReason =+ | Exclude<EvalFailureReason, CustomerEvalFailureReason>+ | (typeof LLM_ERROR_KIND_REASONS)[keyof typeof LLM_ERROR_KIND_REASONS]+ | "unrecoverable"+ | "unexpected";++type LlmEvalTerminalErrorAttribution =+ | { outcome: "platform_error"; reason: EvalPlatformErrorReason }+ | { outcome: "upstream_error" | "customer_error"; reason?: never };++type CustomerEvalFailureReason =+ | typeof EvalFailureReason.InvalidModelConfig+ | typeof EvalFailureReason.InvalidModelOutput;+export const __test = {getEvalScoreCountBucket,};diff --git a/worker/src/features/evaluation/evalFailureErrors.ts b/worker/src/features/evaluation/evalFailureErrors.tsnew file mode 100644index 0000000000..b4b57b3c16--- /dev/null+++ b/worker/src/features/evaluation/evalFailureErrors.ts@@ -0,0 +1,47 @@+import { UnrecoverableError } from "../../errors/UnrecoverableError";++/**+ * Bounded reasons for known evaluation failure sites. Values are emitted as+ * metric tag values, so keep the set small and never derive them from input.+ */+export const EvalFailureReason = {+ InvalidModelConfig: "invalid_model_config",+ InvalidModelOutput: "invalid_model_output",+ ObservationDownload: "observation_download",+ InvalidObservationData: "invalid_observation_data",+ ScorePersistence: "score_persistence",+} as const;++export type EvalFailureReason =+ (typeof EvalFailureReason)[keyof typeof EvalFailureReason];++/** Retryable evaluation failure; BullMQ retries it like any other error. */+export class EvalFailureError extends Error {+ constructor(+ message: string,+ readonly reason: EvalFailureReason,+ ) {+ super(message);+ this.name = "EvalFailureError";+ }+}++/**+ * Terminal evaluation failure. Keeps the `UnrecoverableError` name so existing+ * `isUnrecoverableError` checks continue to stop retries.+ */+export class UnrecoverableEvalFailureError extends UnrecoverableError {+ constructor(+ message: string,+ readonly reason: EvalFailureReason,+ ) {+ super(message);+ }+}++export function getEvalFailureReason(error: unknown): EvalFailureReason | null {+ return error instanceof EvalFailureError ||+ error instanceof UnrecoverableEvalFailureError+ ? error.reason+ : null;+}diff --git a/worker/src/features/evaluation/evalService.ts b/worker/src/features/evaluation/evalService.tsindex b4fd49a703..9b9ecf6843 100644--- a/worker/src/features/evaluation/evalService.ts+++ b/worker/src/features/evaluation/evalService.ts@@ -72,6 +72,10 @@ import { env } from "../../env";import { prisma } from "@langfuse/shared/src/db";import { createW3CTraceId } from "../utils";import { UnrecoverableError } from "../../errors/UnrecoverableError";+import {+ EvalFailureReason,+ UnrecoverableEvalFailureError,+} from "./evalFailureErrors";import { ObservationNotFoundError } from "../../errors/ObservationNotFoundError";import {completeEvalExecution,@@ -1062,8 +1066,9 @@ export async function runLLMAsJudgeEvaluation({logger.warn(`Eval job ${jobExecutionId} will fail. ${modelConfig.error}`,);- throw new UnrecoverableError(+ throw new UnrecoverableEvalFailureError(`Invalid model configuration for job ${jobExecutionId}: ${modelConfig.error}`,+ EvalFailureReason.InvalidModelConfig,);}@@ -1196,8 +1201,9 @@ export async function runLLMAsJudgeEvaluation({if (!parsedLLMOutput.success) {span.setAttribute("eval.execution.outcome", "invalid_model_output");- throw new UnrecoverableError(+ throw new UnrecoverableEvalFailureError(`Invalid LLM response format from model ${modelConfig.config.model}. Error: ${parsedLLMOutput.error}`,+ EvalFailureReason.InvalidModelOutput,);}diff --git a/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts b/worker/src/features/evaluation/observationEval/observationEvalProcessor.tsindex 6c5f3a89ef..f3735854b7 100644--- a/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts+++ b/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts@@ -24,6 +24,11 @@ import {} from "@prisma/client";import { prisma, JobExecutionStatus } from "@langfuse/shared/src/db";import { UnrecoverableError } from "../../../errors/UnrecoverableError";+import {+ EvalFailureError,+ EvalFailureReason,+ UnrecoverableEvalFailureError,+} from "../evalFailureErrors";import { buildEvalExecutionData } from "@langfuse/shared/src/server";import {completeEvalExecution,@@ -174,8 +179,9 @@ export async function processObservationEval();} catch (e) {// S3 download failures are retryable (network issues, temporary unavailability)- throw new Error(+ throw new EvalFailureError(`Failed to download observation from S3 at ${event.observationS3Path}: ${e}`,+ EvalFailureReason.ObservationDownload,);}@@ -185,8 +191,9 @@ export async function processObservationEval(observationData = observationForEvalSchema.parse(parsedJson);} catch (e) {// JSON parse errors are permanent - the data won't change on retry- throw new UnrecoverableError(+ throw new UnrecoverableEvalFailureError(`Invalid observation data from S3 at ${event.observationS3Path}: invalid JSON - ${e}`,+ EvalFailureReason.InvalidObservationData,);}diff --git a/worker/src/queues/__tests__/llmAsJudgeExecutionQueueProcessor.test.ts b/worker/src/queues/__tests__/llmAsJudgeExecutionQueueProcessor.test.tsindex d70836c353..4a3b32f51d 100644--- a/worker/src/queues/__tests__/llmAsJudgeExecutionQueueProcessor.test.ts+++ b/worker/src/queues/__tests__/llmAsJudgeExecutionQueueProcessor.test.ts@@ -9,6 +9,11 @@ import {} from "vitest";import { Job } from "bullmq";import { UnrecoverableError } from "../../errors/UnrecoverableError";+import {+ EvalFailureError,+ EvalFailureReason,+ UnrecoverableEvalFailureError,+} from "../../features/evaluation/evalFailureErrors";const QueueName = {LLMAsJudgeExecution: "llm-as-a-judge-execution-queue",@@ -116,6 +121,7 @@ import { processObservationEval } from "../../features/evaluation/observationEvaimport {LLMAsJudgeExecutionQueue,classifyEvaluatorLlmError,+ logger,recordDistribution,recordIncrement,traceException,@@ -520,6 +526,45 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {expect(traceException).not.toHaveBeenCalled();});++ it.each([+ {+ error: new UnrecoverableEvalFailureError(+ "invalid model configuration",+ EvalFailureReason.InvalidModelConfig,+ ),+ tags: { outcome: "customer_error" },+ },+ {+ error: new UnrecoverableEvalFailureError(+ "invalid JSON",+ EvalFailureReason.InvalidObservationData,+ ),+ tags: { outcome: "platform_error", reason: "invalid_observation_data" },+ },+ {+ error: new UnrecoverableError("Evaluator identity missing"),+ tags: { outcome: "platform_error", reason: "unrecoverable" },+ },+ ])(+ "records and warns terminal failures as $tags.outcome",+ async ({ error, tags }) => {+ (processObservationEval as Mock).mockRejectedValue(error);+ vi.mocked(isUnrecoverableError).mockReturnValue(true);++ await llmAsJudgeExecutionQueueProcessor(createMockJob());++ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.evaluation.execution.terminal",+ 1,+ { evaluator_type: "llm_as_judge", ...tags },+ );+ expect(logger.warn).toHaveBeenCalledWith(+ expect.any(String),+ expect.objectContaining({ projectId, errorName: error.name }),+ );+ },+ );});describe("unexpected errors (retryable by BullMQ)", () => {@@ -568,34 +613,49 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {);});- it("should persist ERROR on the final BullMQ attempt", async () => {- const unexpectedError = new Error("Database connection failed");- (processObservationEval as Mock).mockRejectedValue(unexpectedError);-- await expect(- llmAsJudgeExecutionQueueProcessor(- createMockJob({ attemptsMade: 9, opts: { attempts: 10 } }),+ it.each([+ { error: new Error("Database connection failed"), reason: "unexpected" },+ {+ error: new EvalFailureError(+ "Failed to write score",+ EvalFailureReason.ScorePersistence,),- ).rejects.toThrow(unexpectedError);-- expect(prisma.jobExecution.update).toHaveBeenCalledWith({- where: {- id: jobExecutionId,- projectId,- },- data: {- status: JobExecutionStatus.ERROR,- endTime: expect.any(Date),- error: "An internal error occurred",- executionTraceId: "test-trace-id",- },- });- expect(recordIncrement).toHaveBeenCalledWith(- "langfuse.evaluation.execution.terminal",- 1,- { evaluator_type: "llm_as_judge", outcome: "platform_error" },- );- });+ reason: "score_persistence",+ },+ ])(+ "should persist ERROR on the final BullMQ attempt with reason $reason",+ async ({ error, reason }) => {+ (processObservationEval as Mock).mockRejectedValue(error);++ await expect(+ llmAsJudgeExecutionQueueProcessor(+ createMockJob({ attemptsMade: 9, opts: { attempts: 10 } }),+ ),+ ).rejects.toThrow(error);++ expect(prisma.jobExecution.update).toHaveBeenCalledWith({+ where: {+ id: jobExecutionId,+ projectId,+ },+ data: {+ status: JobExecutionStatus.ERROR,+ endTime: expect.any(Date),+ error: "An internal error occurred",+ executionTraceId: "test-trace-id",+ },+ });+ expect(recordIncrement).toHaveBeenCalledWith(+ "langfuse.evaluation.execution.terminal",+ 1,+ {+ evaluator_type: "llm_as_judge",+ outcome: "platform_error",+ reason,+ },+ );+ },+ );});describe("execution trace ID", () => {diff --git a/worker/src/queues/evalQueue.ts b/worker/src/queues/evalQueue.tsindex 8ad27f4e25..f881390e8c 100644--- a/worker/src/queues/evalQueue.ts+++ b/worker/src/queues/evalQueue.ts@@ -12,6 +12,7 @@ import {QueueJobs,getCurrentSpan,classifyEvaluatorLlmError,+ type EvaluatorLlmErrorClassification,} from "@langfuse/shared/src/server";import { createEvalJobs, evaluate } from "../features/evaluation/evalService";import { processObservationEval } from "../features/evaluation/observationEval";@@ -21,7 +22,7 @@ import { retryObservationNotFound } from "../features/evaluation/retryObservatioimport { isObservationNotFoundError } from "../errors/ObservationNotFoundError";import { env } from "../env";import {- getLlmEvalTerminalErrorOutcome,+ getLlmEvalTerminalErrorAttribution,recordEvalTerminalOutcome,recordEvalTimeToFirstAttempt,} from "../features/evaluation/evalExecutionMetrics";@@ -261,7 +262,13 @@ export const evalJobExecutorQueueProcessorBuilder = (},});- if (llmError || isUnrecoverableError(e)) return;+ if (llmError || isUnrecoverableError(e)) {+ logger.warn(+ `${queueName} job ${job.data.payload.jobExecutionId} failed terminally`,+ getTerminalEvalErrorLogContext(job.data.payload, e, llmError),+ );+ return;+ }traceException(e);logger.error(@@ -385,10 +392,27 @@ export const llmAsJudgeExecutionQueueProcessorBuilder =executionTraceId,},});+ const attribution = getLlmEvalTerminalErrorAttribution({+ error: e,+ classification: llmError,+ isUnrecoverable,+ });recordEvalTerminalOutcome(EvalTemplateType.LLM_AS_JUDGE,- getLlmEvalTerminalErrorOutcome(llmError),+ attribution.outcome,+ attribution.reason,);++ if (isTerminalError || isMediaNotFoundError) {+ logger.warn(+ `LLM-as-Judge execution job ${job.data.payload.jobExecutionId} failed terminally`,+ {+ ...getTerminalEvalErrorLogContext(job.data.payload, e, llmError),+ outcome: attribution.outcome,+ reason: attribution.reason,+ },+ );+ }}if (isTerminalError || (isMediaNotFoundError && isFinalAttempt)) return;@@ -402,3 +426,21 @@ export const llmAsJudgeExecutionQueueProcessorBuilder =throw e;}};++function getTerminalEvalErrorLogContext(+ payload: { projectId: string; jobExecutionId: string },+ error: unknown,+ llmError: EvaluatorLlmErrorClassification | null,+) {+ return {+ projectId: payload.projectId,+ jobExecutionId: payload.jobExecutionId,+ errorName: error instanceof Error ? error.name : typeof error,+ errorMessage: (error instanceof Error+ ? error.message+ : String(error)+ ).slice(0, 500),+ llmErrorKind: llmError?.kind,+ llmStatusCode: llmError?.statusCode,+ };+}