Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

fix(evals): attribute terminal LLM-as-judge failures by reason (#18821)

Tobias Wochinger0793320d5f4b
diff --git a/worker/src/__tests__/evalService.test.ts b/worker/src/__tests__/evalService.test.ts
index 67a52e62fb..76b5d2e46b 100644
--- a/worker/src/__tests__/evalService.test.ts
+++ b/worker/src/__tests__/evalService.test.ts
@@ -50,6 +50,10 @@ import {
generateLlmEvaluatorOutput,
} from "@langfuse/shared/src/server";
import { UnrecoverableError } from "../errors/UnrecoverableError";
+import {
+ EvalFailureReason,
+ UnrecoverableEvalFailureError,
+} from "../features/evaluation/evalFailureErrors";
let OPENAI_API_KEY = process.env.OPENAI_API_KEY;
// Check for both OPENAI_API_KEY and LANGFUSE_LLM_CONNECTION_OPENAI_KEY
@@ -2136,8 +2140,9 @@ Respond with JSON: {"score": <number>, "reasoning": "<explanation>"}`;
};
await expect(evaluate({ event: payload })).rejects.toThrowError(
- new UnrecoverableError(
+ new UnrecoverableEvalFailureError(
`Invalid model configuration for job ${jobExecutionId}: API key for provider "openai" not found in project ${projectId}`,
+ EvalFailureReason.InvalidModelConfig,
),
);
diff --git a/worker/src/features/evaluation/decisionModel/runDecisionModelEvaluation.ts b/worker/src/features/evaluation/decisionModel/runDecisionModelEvaluation.ts
index bc27764b1a..3fdab0906b 100644
--- a/worker/src/features/evaluation/decisionModel/runDecisionModelEvaluation.ts
+++ b/worker/src/features/evaluation/decisionModel/runDecisionModelEvaluation.ts
@@ -25,6 +25,10 @@ import {
type ExtractedVariable,
} from "@langfuse/shared/src/server";
import { UnrecoverableError } from "../../../errors/UnrecoverableError";
+import {
+ EvalFailureReason,
+ UnrecoverableEvalFailureError,
+} from "../evalFailureErrors";
import { createW3CTraceId } from "../../utils";
import { type EvalExecutionResult } from "../evalCompletion";
import { type EvalExecutionDeps } from "../evalExecutionDeps";
@@ -145,8 +149,9 @@ export async function runDecisionModelEvaluation({
logger.warn(
`Eval job ${jobExecutionId} will fail. ${modelConfigError}`,
);
- throw new UnrecoverableError(
+ throw new UnrecoverableEvalFailureError(
`Invalid model configuration for job ${jobExecutionId}: ${modelConfigError}`,
+ EvalFailureReason.InvalidModelConfig,
);
}
@@ -180,7 +185,10 @@ export async function runDecisionModelEvaluation({
} catch (e) {
if (e instanceof DecisionModelEvaluatorError) {
span.setAttribute("eval.execution.outcome", "invalid_model_output");
- throw new UnrecoverableError(e.message);
+ throw new UnrecoverableEvalFailureError(
+ e.message,
+ EvalFailureReason.InvalidModelOutput,
+ );
}
const classification = classifyEvaluatorLlmError(e);
diff --git a/worker/src/features/evaluation/evalCompletion.ts b/worker/src/features/evaluation/evalCompletion.ts
index 12aadde966..7ca48cc2ba 100644
--- a/worker/src/features/evaluation/evalCompletion.ts
+++ b/worker/src/features/evaluation/evalCompletion.ts
@@ -7,6 +7,7 @@ import {
} from "@langfuse/shared/src/server";
import { buildEvalScoreWritePayloads } from "./evalScoreEvent";
import { type EvalExecutionDeps } from "./evalExecutionDeps";
+import { EvalFailureError, EvalFailureReason } from "./evalFailureErrors";
import {
recordEvalScorePersistenceOutcome,
type ObservationEvalExecutionType,
@@ -85,8 +86,9 @@ export async function completeEvalExecution({
);
logger.error(`Failed to persist score: ${e}`, e);
traceException(e);
- throw new Error(
+ throw new EvalFailureError(
`Failed to write score ${jobOutputScoreId} into IngestionQueue`,
+ EvalFailureReason.ScorePersistence,
);
}
diff --git a/worker/src/features/evaluation/evalExecutionMetrics.test.ts b/worker/src/features/evaluation/evalExecutionMetrics.test.ts
index c6df2cf255..90ce2564f5 100644
--- a/worker/src/features/evaluation/evalExecutionMetrics.test.ts
+++ b/worker/src/features/evaluation/evalExecutionMetrics.test.ts
@@ -10,12 +10,13 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
import {
__test,
getCodeEvalTerminalErrorOutcome,
+ getLlmEvalTerminalErrorAttribution,
getLlmEvalTerminalErrorOutcome,
recordEvalScorePersistenceOutcome,
recordEvalTerminalOutcome,
recordEvalTimeToFirstAttempt,
} from "./evalExecutionMetrics";
-
+import { EvalFailureError, EvalFailureReason } from "./evalFailureErrors";
vi.mock("@langfuse/shared/src/server", async (importOriginal) => ({
...(await importOriginal<typeof import("@langfuse/shared/src/server")>()),
recordDistribution: vi.fn(),
@@ -139,6 +140,45 @@ describe("evaluation execution metrics", () => {
},
);
+ it("attributes classified LLM platform errors by error kind", () => {
+ const error = new Error("unexpected AI SDK failure");
+
+ expect(
+ getLlmEvalTerminalErrorAttribution({
+ error,
+ classification: {
+ kind: "ai-sdk",
+ message: error.message,
+ isRetryable: false,
+ error,
+ blockReason: null,
+ },
+ isUnrecoverable: false,
+ }),
+ ).toEqual({ outcome: "platform_error", reason: "llm_ai_sdk" });
+ });
+
+ it("prefers an explicit failure reason over LLM classification", () => {
+ const error = new EvalFailureError(
+ "Failed to download observation: DNS lookup failed",
+ EvalFailureReason.ObservationDownload,
+ );
+
+ expect(
+ getLlmEvalTerminalErrorAttribution({
+ error,
+ classification: {
+ kind: "evaluator-policy",
+ message: error.message,
+ isRetryable: false,
+ error,
+ blockReason: null,
+ },
+ isUnrecoverable: false,
+ }),
+ ).toEqual({ outcome: "platform_error", reason: "observation_download" });
+ });
+
it.each([
{
code: CodeEvalDispatcherErrorCodes.USER_CODE_ERROR,
diff --git a/worker/src/features/evaluation/evalExecutionMetrics.ts b/worker/src/features/evaluation/evalExecutionMetrics.ts
index 8a7cbcc7a1..fc39ee89e9 100644
--- a/worker/src/features/evaluation/evalExecutionMetrics.ts
+++ b/worker/src/features/evaluation/evalExecutionMetrics.ts
@@ -6,6 +6,7 @@ import {
recordIncrement,
type EvaluatorLlmErrorClassification,
} from "@langfuse/shared/src/server";
+import { EvalFailureReason, getEvalFailureReason } from "./evalFailureErrors";
export type ObservationEvalExecutionType =
| typeof EvalTemplateType.LLM_AS_JUDGE
@@ -18,6 +19,21 @@ export type EvalExecutionTerminalOutcome =
| "customer_error"
| "cancelled";
+const CUSTOMER_EVAL_FAILURE_REASONS: ReadonlySet<EvalFailureReason> =
+ new Set<CustomerEvalFailureReason>([
+ EvalFailureReason.InvalidModelConfig,
+ EvalFailureReason.InvalidModelOutput,
+ ]);
+
+const LLM_ERROR_KIND_REASONS = {
+ provider: "llm_provider",
+ validation: "llm_validation",
+ "ai-sdk": "llm_ai_sdk",
+ timeout: "llm_timeout",
+ abort: "llm_abort",
+ "evaluator-policy": "llm_evaluator_policy",
+} as const satisfies Record<EvaluatorLlmErrorClassification["kind"], string>;
+
type EvalScoreCountBucket = "0" | "1" | "2" | "3-5" | "6-10" | "11+";
const LLM_EVAL_CUSTOMER_OUTPUT_ERROR_NAMES: ReadonlySet<string> = new Set([
@@ -53,13 +69,49 @@ export function recordEvalTimeToFirstAttempt(
export function recordEvalTerminalOutcome(
executionType: ObservationEvalExecutionType,
outcome: EvalExecutionTerminalOutcome,
+ platformErrorReason?: EvalPlatformErrorReason,
): void {
recordIncrement("langfuse.evaluation.execution.terminal", 1, {
evaluator_type: getEvaluatorTypeTag(executionType),
outcome,
+ ...(outcome === "platform_error" && platformErrorReason
+ ? { reason: platformErrorReason }
+ : {}),
});
}
+/**
+ * Attributes a terminal LLM evaluator failure. Known failure sites carry an
+ * explicit reason; everything else falls back to the LLM classification, then
+ * to whether the error was raised as unrecoverable.
+ */
+export function getLlmEvalTerminalErrorAttribution(params: {
+ error: unknown;
+ classification: EvaluatorLlmErrorClassification | null;
+ isUnrecoverable: boolean;
+}): LlmEvalTerminalErrorAttribution {
+ const { error, classification, isUnrecoverable } = params;
+
+ const failureReason = getEvalFailureReason(error);
+ if (failureReason) {
+ return isCustomerEvalFailureReason(failureReason)
+ ? { outcome: "customer_error" }
+ : { outcome: "platform_error", reason: failureReason };
+ }
+
+ if (classification) {
+ const outcome = getLlmEvalTerminalErrorOutcome(classification);
+ return outcome === "platform_error"
+ ? { outcome, reason: LLM_ERROR_KIND_REASONS[classification.kind] }
+ : { outcome };
+ }
+
+ return {
+ outcome: "platform_error",
+ reason: isUnrecoverable ? "unrecoverable" : "unexpected",
+ };
+}
+
export function recordEvalScorePersistenceOutcome(
executionType: ObservationEvalExecutionType,
outcome: "success" | "failure",
@@ -163,6 +215,26 @@ function getEvalScoreCountBucket(scoreCount: number): EvalScoreCountBucket {
return "11+";
}
+function isCustomerEvalFailureReason(
+ reason: EvalFailureReason,
+): reason is CustomerEvalFailureReason {
+ return CUSTOMER_EVAL_FAILURE_REASONS.has(reason);
+}
+
+type EvalPlatformErrorReason =
+ | Exclude<EvalFailureReason, CustomerEvalFailureReason>
+ | (typeof LLM_ERROR_KIND_REASONS)[keyof typeof LLM_ERROR_KIND_REASONS]
+ | "unrecoverable"
+ | "unexpected";
+
+type LlmEvalTerminalErrorAttribution =
+ | { outcome: "platform_error"; reason: EvalPlatformErrorReason }
+ | { outcome: "upstream_error" | "customer_error"; reason?: never };
+
+type CustomerEvalFailureReason =
+ | typeof EvalFailureReason.InvalidModelConfig
+ | typeof EvalFailureReason.InvalidModelOutput;
+
export const __test = {
getEvalScoreCountBucket,
};
diff --git a/worker/src/features/evaluation/evalFailureErrors.ts b/worker/src/features/evaluation/evalFailureErrors.ts
new file mode 100644
index 0000000000..b4b57b3c16
--- /dev/null
+++ b/worker/src/features/evaluation/evalFailureErrors.ts
@@ -0,0 +1,47 @@
+import { UnrecoverableError } from "../../errors/UnrecoverableError";
+
+/**
+ * Bounded reasons for known evaluation failure sites. Values are emitted as
+ * metric tag values, so keep the set small and never derive them from input.
+ */
+export const EvalFailureReason = {
+ InvalidModelConfig: "invalid_model_config",
+ InvalidModelOutput: "invalid_model_output",
+ ObservationDownload: "observation_download",
+ InvalidObservationData: "invalid_observation_data",
+ ScorePersistence: "score_persistence",
+} as const;
+
+export type EvalFailureReason =
+ (typeof EvalFailureReason)[keyof typeof EvalFailureReason];
+
+/** Retryable evaluation failure; BullMQ retries it like any other error. */
+export class EvalFailureError extends Error {
+ constructor(
+ message: string,
+ readonly reason: EvalFailureReason,
+ ) {
+ super(message);
+ this.name = "EvalFailureError";
+ }
+}
+
+/**
+ * Terminal evaluation failure. Keeps the `UnrecoverableError` name so existing
+ * `isUnrecoverableError` checks continue to stop retries.
+ */
+export class UnrecoverableEvalFailureError extends UnrecoverableError {
+ constructor(
+ message: string,
+ readonly reason: EvalFailureReason,
+ ) {
+ super(message);
+ }
+}
+
+export function getEvalFailureReason(error: unknown): EvalFailureReason | null {
+ return error instanceof EvalFailureError ||
+ error instanceof UnrecoverableEvalFailureError
+ ? error.reason
+ : null;
+}
diff --git a/worker/src/features/evaluation/evalService.ts b/worker/src/features/evaluation/evalService.ts
index b4fd49a703..9b9ecf6843 100644
--- a/worker/src/features/evaluation/evalService.ts
+++ b/worker/src/features/evaluation/evalService.ts
@@ -72,6 +72,10 @@ import { env } from "../../env";
import { prisma } from "@langfuse/shared/src/db";
import { createW3CTraceId } from "../utils";
import { UnrecoverableError } from "../../errors/UnrecoverableError";
+import {
+ EvalFailureReason,
+ UnrecoverableEvalFailureError,
+} from "./evalFailureErrors";
import { ObservationNotFoundError } from "../../errors/ObservationNotFoundError";
import {
completeEvalExecution,
@@ -1062,8 +1066,9 @@ export async function runLLMAsJudgeEvaluation({
logger.warn(
`Eval job ${jobExecutionId} will fail. ${modelConfig.error}`,
);
- throw new UnrecoverableError(
+ throw new UnrecoverableEvalFailureError(
`Invalid model configuration for job ${jobExecutionId}: ${modelConfig.error}`,
+ EvalFailureReason.InvalidModelConfig,
);
}
@@ -1196,8 +1201,9 @@ export async function runLLMAsJudgeEvaluation({
if (!parsedLLMOutput.success) {
span.setAttribute("eval.execution.outcome", "invalid_model_output");
- throw new UnrecoverableError(
+ throw new UnrecoverableEvalFailureError(
`Invalid LLM response format from model ${modelConfig.config.model}. Error: ${parsedLLMOutput.error}`,
+ EvalFailureReason.InvalidModelOutput,
);
}
diff --git a/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts b/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts
index 6c5f3a89ef..f3735854b7 100644
--- a/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts
+++ b/worker/src/features/evaluation/observationEval/observationEvalProcessor.ts
@@ -24,6 +24,11 @@ import {
} from "@prisma/client";
import { prisma, JobExecutionStatus } from "@langfuse/shared/src/db";
import { UnrecoverableError } from "../../../errors/UnrecoverableError";
+import {
+ EvalFailureError,
+ EvalFailureReason,
+ UnrecoverableEvalFailureError,
+} from "../evalFailureErrors";
import { buildEvalExecutionData } from "@langfuse/shared/src/server";
import {
completeEvalExecution,
@@ -174,8 +179,9 @@ export async function processObservationEval(
);
} catch (e) {
// S3 download failures are retryable (network issues, temporary unavailability)
- throw new Error(
+ throw new EvalFailureError(
`Failed to download observation from S3 at ${event.observationS3Path}: ${e}`,
+ EvalFailureReason.ObservationDownload,
);
}
@@ -185,8 +191,9 @@ export async function processObservationEval(
observationData = observationForEvalSchema.parse(parsedJson);
} catch (e) {
// JSON parse errors are permanent - the data won't change on retry
- throw new UnrecoverableError(
+ throw new UnrecoverableEvalFailureError(
`Invalid observation data from S3 at ${event.observationS3Path}: invalid JSON - ${e}`,
+ EvalFailureReason.InvalidObservationData,
);
}
diff --git a/worker/src/queues/__tests__/llmAsJudgeExecutionQueueProcessor.test.ts b/worker/src/queues/__tests__/llmAsJudgeExecutionQueueProcessor.test.ts
index d70836c353..4a3b32f51d 100644
--- a/worker/src/queues/__tests__/llmAsJudgeExecutionQueueProcessor.test.ts
+++ b/worker/src/queues/__tests__/llmAsJudgeExecutionQueueProcessor.test.ts
@@ -9,6 +9,11 @@ import {
} from "vitest";
import { Job } from "bullmq";
import { UnrecoverableError } from "../../errors/UnrecoverableError";
+import {
+ EvalFailureError,
+ EvalFailureReason,
+ UnrecoverableEvalFailureError,
+} from "../../features/evaluation/evalFailureErrors";
const QueueName = {
LLMAsJudgeExecution: "llm-as-a-judge-execution-queue",
@@ -116,6 +121,7 @@ import { processObservationEval } from "../../features/evaluation/observationEva
import {
LLMAsJudgeExecutionQueue,
classifyEvaluatorLlmError,
+ logger,
recordDistribution,
recordIncrement,
traceException,
@@ -520,6 +526,45 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {
expect(traceException).not.toHaveBeenCalled();
});
+
+ it.each([
+ {
+ error: new UnrecoverableEvalFailureError(
+ "invalid model configuration",
+ EvalFailureReason.InvalidModelConfig,
+ ),
+ tags: { outcome: "customer_error" },
+ },
+ {
+ error: new UnrecoverableEvalFailureError(
+ "invalid JSON",
+ EvalFailureReason.InvalidObservationData,
+ ),
+ tags: { outcome: "platform_error", reason: "invalid_observation_data" },
+ },
+ {
+ error: new UnrecoverableError("Evaluator identity missing"),
+ tags: { outcome: "platform_error", reason: "unrecoverable" },
+ },
+ ])(
+ "records and warns terminal failures as $tags.outcome",
+ async ({ error, tags }) => {
+ (processObservationEval as Mock).mockRejectedValue(error);
+ vi.mocked(isUnrecoverableError).mockReturnValue(true);
+
+ await llmAsJudgeExecutionQueueProcessor(createMockJob());
+
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.evaluation.execution.terminal",
+ 1,
+ { evaluator_type: "llm_as_judge", ...tags },
+ );
+ expect(logger.warn).toHaveBeenCalledWith(
+ expect.any(String),
+ expect.objectContaining({ projectId, errorName: error.name }),
+ );
+ },
+ );
});
describe("unexpected errors (retryable by BullMQ)", () => {
@@ -568,34 +613,49 @@ describe("llmAsJudgeExecutionQueueProcessor", () => {
);
});
- it("should persist ERROR on the final BullMQ attempt", async () => {
- const unexpectedError = new Error("Database connection failed");
- (processObservationEval as Mock).mockRejectedValue(unexpectedError);
-
- await expect(
- llmAsJudgeExecutionQueueProcessor(
- createMockJob({ attemptsMade: 9, opts: { attempts: 10 } }),
+ it.each([
+ { error: new Error("Database connection failed"), reason: "unexpected" },
+ {
+ error: new EvalFailureError(
+ "Failed to write score",
+ EvalFailureReason.ScorePersistence,
),
- ).rejects.toThrow(unexpectedError);
-
- expect(prisma.jobExecution.update).toHaveBeenCalledWith({
- where: {
- id: jobExecutionId,
- projectId,
- },
- data: {
- status: JobExecutionStatus.ERROR,
- endTime: expect.any(Date),
- error: "An internal error occurred",
- executionTraceId: "test-trace-id",
- },
- });
- expect(recordIncrement).toHaveBeenCalledWith(
- "langfuse.evaluation.execution.terminal",
- 1,
- { evaluator_type: "llm_as_judge", outcome: "platform_error" },
- );
- });
+ reason: "score_persistence",
+ },
+ ])(
+ "should persist ERROR on the final BullMQ attempt with reason $reason",
+ async ({ error, reason }) => {
+ (processObservationEval as Mock).mockRejectedValue(error);
+
+ await expect(
+ llmAsJudgeExecutionQueueProcessor(
+ createMockJob({ attemptsMade: 9, opts: { attempts: 10 } }),
+ ),
+ ).rejects.toThrow(error);
+
+ expect(prisma.jobExecution.update).toHaveBeenCalledWith({
+ where: {
+ id: jobExecutionId,
+ projectId,
+ },
+ data: {
+ status: JobExecutionStatus.ERROR,
+ endTime: expect.any(Date),
+ error: "An internal error occurred",
+ executionTraceId: "test-trace-id",
+ },
+ });
+ expect(recordIncrement).toHaveBeenCalledWith(
+ "langfuse.evaluation.execution.terminal",
+ 1,
+ {
+ evaluator_type: "llm_as_judge",
+ outcome: "platform_error",
+ reason,
+ },
+ );
+ },
+ );
});
describe("execution trace ID", () => {
diff --git a/worker/src/queues/evalQueue.ts b/worker/src/queues/evalQueue.ts
index 8ad27f4e25..f881390e8c 100644
--- a/worker/src/queues/evalQueue.ts
+++ b/worker/src/queues/evalQueue.ts
@@ -12,6 +12,7 @@ import {
QueueJobs,
getCurrentSpan,
classifyEvaluatorLlmError,
+ type EvaluatorLlmErrorClassification,
} from "@langfuse/shared/src/server";
import { createEvalJobs, evaluate } from "../features/evaluation/evalService";
import { processObservationEval } from "../features/evaluation/observationEval";
@@ -21,7 +22,7 @@ import { retryObservationNotFound } from "../features/evaluation/retryObservatio
import { isObservationNotFoundError } from "../errors/ObservationNotFoundError";
import { env } from "../env";
import {
- getLlmEvalTerminalErrorOutcome,
+ getLlmEvalTerminalErrorAttribution,
recordEvalTerminalOutcome,
recordEvalTimeToFirstAttempt,
} from "../features/evaluation/evalExecutionMetrics";
@@ -261,7 +262,13 @@ export const evalJobExecutorQueueProcessorBuilder = (
},
});
- if (llmError || isUnrecoverableError(e)) return;
+ if (llmError || isUnrecoverableError(e)) {
+ logger.warn(
+ `${queueName} job ${job.data.payload.jobExecutionId} failed terminally`,
+ getTerminalEvalErrorLogContext(job.data.payload, e, llmError),
+ );
+ return;
+ }
traceException(e);
logger.error(
@@ -385,10 +392,27 @@ export const llmAsJudgeExecutionQueueProcessorBuilder =
executionTraceId,
},
});
+ const attribution = getLlmEvalTerminalErrorAttribution({
+ error: e,
+ classification: llmError,
+ isUnrecoverable,
+ });
recordEvalTerminalOutcome(
EvalTemplateType.LLM_AS_JUDGE,
- getLlmEvalTerminalErrorOutcome(llmError),
+ attribution.outcome,
+ attribution.reason,
);
+
+ if (isTerminalError || isMediaNotFoundError) {
+ logger.warn(
+ `LLM-as-Judge execution job ${job.data.payload.jobExecutionId} failed terminally`,
+ {
+ ...getTerminalEvalErrorLogContext(job.data.payload, e, llmError),
+ outcome: attribution.outcome,
+ reason: attribution.reason,
+ },
+ );
+ }
}
if (isTerminalError || (isMediaNotFoundError && isFinalAttempt)) return;
@@ -402,3 +426,21 @@ export const llmAsJudgeExecutionQueueProcessorBuilder =
throw e;
}
};
+
+function getTerminalEvalErrorLogContext(
+ payload: { projectId: string; jobExecutionId: string },
+ error: unknown,
+ llmError: EvaluatorLlmErrorClassification | null,
+) {
+ return {
+ projectId: payload.projectId,
+ jobExecutionId: payload.jobExecutionId,
+ errorName: error instanceof Error ? error.name : typeof error,
+ errorMessage: (error instanceof Error
+ ? error.message
+ : String(error)
+ ).slice(0, 500),
+ llmErrorKind: llmError?.kind,
+ llmStatusCode: llmError?.statusCode,
+ };
+}